{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import torch as t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ReLU + Sigmoid + Cross Entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(28*28, 30)\n",
    "        self.fc2 = nn.Linear(30, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.logsigmoid(self.fc2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mnist_loader import load_data_shared, vectorized_result\n",
    "training_data, validation_data, test_data = load_data_shared(filename=\"../mnist.pkl.gz\",\n",
    "                                                                     seed=666,\n",
    "                                                                     train_size=2000,\n",
    "                                                                     vali_size=0,\n",
    "                                                                     test_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(data, net):\n",
    "    with t.no_grad():\n",
    "        #for index in range(test_data[0].shape[0]):\n",
    "            # get the inputs\n",
    "        inputs, labels = t.Tensor(data[0]), data[1]\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        _, predicted = t.max(outputs, 1)\n",
    "\n",
    "        #print('Predicted: ', predicted)\n",
    "        #print('target: ', t.Tensor(labels).int())\n",
    "\n",
    "        correct = (predicted == t.Tensor(labels)).sum().item()\n",
    "        accuracy = correct / data[0].shape[0]\n",
    "        return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(net, criterion, optimizer):\n",
    "    loss_scores = []\n",
    "    test_scores = []\n",
    "    train_scores = []\n",
    "    for epoch in range(100):  # loop over the dataset multiple times\n",
    "\n",
    "        # get the inputs\n",
    "        inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n",
    "        vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels.long())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        loss_scores.append(loss.item())\n",
    "        train_scores.append(predict(training_data, net))\n",
    "        test_scores.append(predict(test_data, net))\n",
    "    print('Finished Training')\n",
    "    return loss_scores, train_scores, test_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net = Net()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr = 1e-1)\n",
    "loss_scores, train_scores, test_scores = fit(net, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deVxVdf7H8dcHEBRxxxVFXFBz13ApbdG0bBm1tFzS0hbbnMqmmubXtNneNFPNjDU5pZmWmrZZWamlk1Yq4I644MqmgAsIyP75/XExESGvevFyL5/n48FDzjnfc+/ndODd4Xu/33NEVTHGGOP5fNxdgDHGGNewQDfGGC9hgW6MMV7CAt0YY7yEBboxxngJP3e9cXBwsIaFhbnr7Y0xxiNFR0enqWrDsra5LdDDwsKIiopy19sbY4xHEpF95W2zLhdjjPESFujGGOMlLNCNMcZLWKAbY4yXsEA3xhgvYYFujDFewgLdGGO8hNvGoRtjTJWQfxyiZsDxoyfXtR8CIRe7/K0s0I0xpqIcPwJzx8D+XwE5ub5WEwt0Y4zxGOmJMGcEHN4FI2dC55sq/C0t0I0x5lzt/Rl+fhMK807flhILedlw60JofcUFKccC3RhjzkXMF/DZ3RAYDHVbnL69UUcY/Bw07XbBSrJAN8aYs7X2v7D4MWjRG8bMg8D67q4IsEA3xhjnqcLyl+Cn16DdtTByBvgHuruq31igG2OMMwoL4JtHYN0s6DEebngTfJ2L0Jz8QuJSMtl24BjbkjO4rmtTeobWc3mJFujGmKotP8cxEqUUVSUrr5Cc/EJy8gqovfo1au9byt5O97Ot9YMQm+bYvbCIQ5m5pGXmcSgrj6zcArLzCsnKLeBQlmP9kew8VB2vG+DnQ3jjIPcFuogMAd4CfIH3VPWVUttDgVlA3eI2T6jqYhfXaowxrlVUCDOugeQNp20SIKj4C6BIhacKJjA7uj9ErzutvY9A/Zr+1AzwI9Dfj5r+vrQKrkmvsPo0rBVAeKNadGhai7AGNfH1kdP2d4UzBrqI+ALTgMFAAhApIotUdWuJZn8FPlHVd0SkI7AYCKuAeo0xxnU2fAzJGzje7wnWZjZkzZ5D7E7LRgTaNQqiY7M6BPr7EuDnQ2GdUIY06caNxctSPFHIz1doUNOfeoH++FRQUDvLmSv03kCcqu4GEJF5wDCgZKArULv4+zpAkiuLNMYYV1JVdiQcpPl3z5LsfxFDlneloAjaN27HjdeEMKx7M5rWqeHuMs+aM4EeAsSXWE4A+pRq8yywRET+CNQEBpX1QiIyCZgEEBoaera1GmPMOTmYkcPPcWlsTkwnNjmDbQeOcVvufB6plsZbgY9yR//WDO8eQsdmtc/8YpWYM4Fe1t8QWmp5DPCBqv5dRC4BZotIZ1UtOmUn1enAdICIiIjSr2GMMS6RnVfAmj2HWbUzjZU7U9lxMBOAGtV8ad+kFre09+OPO77heOsb+Net97m5WtdxJtATgJLToJpzepfKncAQAFX9VUSqA8FAiiuKNMaY33M0O4/1+48Sve8IkXsPs37/UfIKi/D386F3WH1u6tmc/m2DuahpbXwF+Pxe0AKqDZnq7tJdyplAjwTCRaQVkAiMBsaWarMfuAr4QEQuAqoDqa4s1BhjTsjOK2DD/qOsjEtj1c40tiSlowq+PsJFTWsxoV8Y/dsG07tVfapX8z25Y2EBfPUQbJoHl/0JGrRx30FUgDMGuqoWiMhk4HscQxJnqGqMiEwFolR1EfAn4L8iMgVHd8wEVbUuFWPMecnMLWDdviOkHMslLTOXpKPHWb//KFuTMygsUvx8hB6hdXn4qnb0blWfbi3qEOhfTqzlZcPCO2DHt3D54zDg/y7swVwA4q7cjYiI0KioKLe8tzHGBTJT4cv74fDusreHXw2Dnz85m7KoEH6YCukJ8Ic3IaDWybZr3oXI90CLKCjS3ybmHM8vpGRE+QgE+PlSvZoPAdV8qVHNBx9xcqhgTgZkpcL1r0Ovu87tmCsBEYlW1YiyttlMUWPM2Tu8B+bcBBnJ0P5aKB2qucdg9dtwZC+MeB98fB13Jtz6JSCOmZljF0BgA7IX/5XAqGnE1+xMXG49juUWABDo70uTBjVoXCugeCy4L36+5zPOW6DrLdDumvN4jcrNAt0YU7a8bCjIOX39oV0wbywU5cPtixx3HCzLiTsSzh4Ovv6wdyVc/SLaoA36yQSO/OtKNhSEcVXhKj4sGMzL6ROJaBXMZeHBXBbekA5NaiHOXn0bwALdGFOWDR87Pjws68ENALWbw4SvoWH78l+j991QsyF8djeqRSQMeIsFmZfw1S/J1Mt+gvf9X+cqWUVUq/vpdMWjrA+pe+oHmOasWaAbY05SdTyBZ9mzEHYZdLjh9DY+vnDRUKjV+LRNKRk5LNl6kKVbD3IoK5dA/8Z0aPQWSYePsezbhvhIHH1aNeDGK0bi1+IGyEogos2Aij+uKsIC3RjjUFQES5509H13HgHD3wG/gDPutictiyUxB/g+5gDr44+iCmENAmkVXJPsvELW54XSuFkArw1qwlUXNaJBUMnXDK+446mCLNCNMVCQC1/cB1s+hT73wTUvgY9Puc0zcvL5cn0iH6+NJzY5A4DOIbWZMqgd13RqQrvGQdb/7QYW6MZUdTkZMH8c7PkfDHoW+j182qiVjJx8YpMyiE3OYFNCOt/FHCA7r5BOzWrz1A0duaZTY5rXqzxP7qmqLNCNqcoyU2DOCDgY4+hi6X7qJPCtSRm8t3I3izYmUVDkGBBeL7Aa13dpyri+LenavI5diVciFujGVFWHd8PsmyDzoONBx+2uBuB4XiFLth7gk6h4fo47RKC/L+P6tuSK9g3p2LQ2jWoFWIhXUhboxlRFSRvgo5GO2Zu3fwXNIziSlcdLi2NZvDmZrLxCQurW4M9DOjC2dyh1Aqu5u2LjBAt0Y6qChChY/iIU5juWk9ZDjXow/nMIDif9eD7jZ6xhx8FMbuwewo09Q+gdVt/tT+AxZ8cC3RhvV5jvuF3s8cMQXDwRqM1AuPZVqN2MzNwCJsxcy/YDx5g+PoIBHRq5t15zzizQjfF262bBoZ0wei50uO6UTZm5Bdw1K5JNCelMG9vTwtzDWaAb481yMmDFK9Cyn+MmWiUsiTnAM4tiOJiRwxujujOkcxM3FWlcxQLdGG/281uOW8aOnf/b2PJDmbk8+fkWvos5QIcmtXj71p70CK3n5kKNK1igG+Ot0hPh139Dl5sh5GIA4lKOMfGDSA5m5PL4kPbcfVlrqvmWPyPUeBYLdGO81eq3QYtg4FMA/BKXxj1zognw82XBPZfQrUVdNxdoXM0C3RhvtXOJ446J9Vry+foEHluwidYNazJjQi+bpu+l7G8tY7zR0XhI2wFtBjJ79T6mzN9I71b1WXjfpRbmXsypQBeRISKyXUTiROSJMra/ISIbir92iMhR15dqjHHarh8A+ORoO576YguDLmrEjAm9qF3dZnx6szN2uYiILzANGAwkAJEiskhVt55oo6pTSrT/I9CjAmo1xjhr149k+jfi8Z/yuKFrM94Y1d0+/KwCnDnDvYE4Vd2tqnnAPGDY77QfA8x1RXHGmHNQWEDezuV8k92R4d1DeGt0DwvzKsKZsxwCxJdYTihedxoRaQm0An4sZ/skEYkSkajU1NSzrdUY44T1q3/APz+DlEb9eG1kN3ztfixVhjOBXtZPg5bTdjSwUFULy9qoqtNVNUJVIxo2bOhsjcYYJ8UkpfPLkk8oxIcJ4yfg72dX5lWJM2c7AWhRYrk5kFRO29FYd4sxblFQWMQj8zdyuc8mCpt0p1Y9uy9LVeNMoEcC4SLSSkT8cYT2otKNRKQ9UA/41bUlGmOc8dGa/SQfTKazxuHffrC7yzFucMZAV9UCYDLwPRALfKKqMSIyVUSGlmg6BpinquV1xxhjKkJREUdSEpm5ZC2Tm25HKHLcHtdUOeKu/I2IiNCoqCi3vLcxXiMzBT4eBUnrTq6rXgce2w2+NhHcG4lItKpGlLXNzrgxnqr4maBFGcm8XjCaTq1CuL5LU2jSxcK8irKzbownSt4Ic0ZQVFjAlOrP85OGsWLMALBnf1ZpFujGeBpV+PQuinz9eaDa8/yQVpcZt/e0BzkbuzmXMR4nMRrSdvAfbmZpSh3eubUn/cOD3V2VqQTsCt0YD3NszWz88efdtC78a0wPrrqosbtLMpWEXaEb40G+WbeXos0L+EF78erYflzbpam7SzKViF2hG+MBcgsKeebLGI5Ef8r1/ln0Gv4ADTtbmJtTWaAbU8kdzMjhntnRbIg/ytKm0Wh+Exp2vcbdZZlKyLpcjKnE1u8/wh/+tYodB4/x/sgwwtN/RbrebOPMTZks0I2ppL7amMSo6avx9/Ph0/su5aqClVBUAN3Gurs0U0nZ/+aNqWRUlbdX7GL+kp9YWvMtQnMTkP/iCPMmXaFxR3eXaCopC3RjKpHsvAKe/jKG2HUr+SbwdYKqCXLxH0GK/5jucL17CzSVmgW6MRVJFTKSoE4ZD/k6vBuOn3yeekxyBm8t20m1Y/v5vMZ7VAuqj4z/AoLDL2DBxpNZoBtTUQoL4JtHYN0s6D8FrnoGRBwhv+oN+OG5U5p3AqYDVAOCO8G4T6G2DU00zrNAN6Yi5B+HhXfC9m8gJMIR4JmpcMMbsPQpWPMf6DwCutzC5xsS+WpjEle0C2ZURCjVA/wh9BIICHL3URgPY4FujKvlpDvuUb5/NVz7N+h9N/zvVVjxMuz6AY4lQ98H4OoX+GbLQaasL+TmiyO4bWRXROyBzubcWaAb42o/vgDxa2HkDOh8k2PdlU9AUCP47i8weCpc+iBbkjL404IN9Aytyws3drYwN+fNAt0YV0rbCVEz4OLbT4b5CRF3QI/bwNePfYeymPRhFPUC/fnP+IsJ8PN1T73Gq9jEImPO1YEtsOIVKMg9uW7Zs+BXHa78S9n7+PoRtfcwN779C9n5hfz3tgga1ap+Qco13s+pQBeRISKyXUTiROSJctrcIiJbRSRGRD52bZnGVELLnnH0i88Z4eg33/szbPsa+j/s6F4pw5cbEhn73hrq1KjG5/f3o3NInQtctPFmZ+xyERFfYBowGEgAIkVkkapuLdEmHPgL0E9Vj4hI2T/NxniLjGTY9SOEXgr7f4WZ1zuGJNZq5vjAsxRVZdryOF5fsoPeYfV5d/zF1Kvp74bCjTdzpg+9NxCnqrsBRGQeMAzYWqLN3cA0VT0CoKopri7UmEpl8wLQIhj6Tzi6D+bfBvlZMPwd8A88pWleQRH/9/lmFkYnMLx7M14d2dX6zE2FcCbQQ4D4EssJQJ9SbdoBiMjPgC/wrKp+V/qFRGQSMAkgNDT0XOo1xv1UYeNcaN7LMYszOBzu+BbilkHXUac0zcjJ554Po/l19yEeHhTOQ1eF22gWU2GcCfSyfvq0jNcJB64EmgMrRaSzqh49ZSfV6RRPhouIiCj9GsZ4hgObIGUrXP/3k+uadnN8lZCTX8hds6JYt+8Ib4zqxo09ml/gQk1V48yHoglAixLLzYGkMtp8qar5qroH2I4j4I3xPhvmgq8/dLqp3CYFhUVM/ngdkXsP8/dbLMzNheFMoEcC4SLSSkT8gdHAolJtvgAGAIhIMI4umN2uLNSYSqEw39F/3m4IBNYvs0lRkfLnTzezLDaFqUM7Max7GTfmMqYCnLHLRVULRGQy8D2O/vEZqhojIlOBKFVdVLztahHZChQCj6nqoYos3Bi3iFsG2WnQveyHTBzOyuPxhRtZFpvCI4PbMf6SsAtbn6nSRNU9XdkREREaFRXllvc25pykbofZN0FRPkyJAd9qp2z+OS6NKfM3cDQ7nyeu7cDEfmH2AahxORGJVtWIsrbZ1H9jnLF/DXx8i6PvfNzC08J8zup9PPXlFloH12TmxF50amYThsyFZ4FuzJls/w4WTHDcm3zcZ1C/1Smbv485wFNfbmFA+0b8e2wPAv3t18q4h93LxZjfs242zBsLDdvDHUtOC/PofYd5cO56ujWvy7SxPS3MjVtZoBtTFlX46XVYNBlaXwETvoGghqc0iUvJ5M5ZUTStU533b4+ghr/N/jTuZZcTxoBjOOLSZyCx+IP6/OOOCURdboFh08Dv1PuubIw/ysQPIvHzEWbd0ZsGQQFuKNqYU9kVujG5mY4nDK2eBj5+UK2GY4z5wL/Cje+eFuYrtqcwevpqagb4suDeS2nZoKabCjfmVHaFbqq2rDT46GZI3gB/+KfjwRTlUFXmrN7Hc19tpV3jWnxwRy+7l7mpVCzQTdWSlw0zh0DypuIV6nggxaiPoMN15e6WnH6cxxduYuXONC5v15BpY3tQq3q1ctsb4w4W6KZqWT0NkjdC3/vBP8ixrsP10Kx7mc1TMnL4YkMi//oxjoJC5flhnbi1T0t8fGzCkKl8LNBN1ZGZAqvehA43wJCXf7fpiu0pzPh5L6t2plKkcEnrBrx8UxfCgq2/3FReFuim6ljxMhTkwKDnym2Sk1/Ii9/EMnv1PkLq1uD+K9syvEcIbRsFXcBCjTk3FuimakjdDtGzoNddENy2zCbbDxzjwbnr2X7wGHf1b8VjQ9rbk4WMR7FAN54tLxs2zYf87N9vF/sV+NeEK/5c5uaF0Qn89YvNBAX48cHEXlzZ3h6LazyPBbrxXNmHHePHE9Y60Vjgur9BzQanrM3JL+TZRTHMi4ynb+v6/HNMDxuKaDyWBbrxTEfjYc4IOLIXbv4A2gz8/fbiCwEn+8FVlRU7Unl5cSw7DmZy/5VteGRwO/x8ba6d8VwW6MbzpMQ67kuelwXjP4Ow/me1+5bEdF5aHMsvuw4RWj+QmRN6MaCDdbEYz2eBbjzLvl9h7ijwqwF3fAuNOzm9q6ry4a/7mPr1VurUqMazf+jI2D4t8fezq3LjHSzQjefY9g0svAPqtHBcmdcNdXrX3IJCnv4ihvlR8Qy6qBF/v6U7dWrYTE/jXZy6NBGRISKyXUTiROSJMrZPEJFUEdlQ/HWX60s1VVr0BzB/HDTuDHd8f1Zhvu9QFqOnr2Z+VDx/HNiW6eMjLMyNVzrjFbqI+ALTgMFAAhApIotUdWuppvNVdXIF1GiqMlX432uw4iVoOxhumeUYfujUrsqcNft5eXEsviJMG9uT67s2reCCjXEfZ7pcegNxqrobQETmAcOA0oFujGsVFcLixyDqfeg2Fob+87RneZYnLTOXKfM3sHJnGpeFB/PqiK40q1ujggs2xr2cCfQQIL7EcgLQp4x2I0TkcmAHMEVV48toY4xz8nPgs7shdhH0exgGPQvi3A2xtiSmc8/saNIyc3lheGdu7ROKOLmvMZ7MmT70sn4TtNTyV0CYqnYFlgGzynwhkUkiEiUiUampqWdXqak6jh91jDGPXQTXvAyDn3M6zL/amMTI//xCkSoL772UcX1bWpibKsOZK/QEoEWJ5eZAUskGqnqoxOJ/gVfLeiFVnQ5MB4iIiCj9PwVTVR07CPt+dnyvRbDqDce9V0a8D11GOvcSOfm8+E0s8yLjiWhZj3fGXUzDWvZYOFO1OBPokUC4iLQCEoHRwNiSDUSkqaomFy8OBWJdWqXxXskbYc5IyEo5uc4/CG795MyzP4v9EpfGYws3kZx+nHuvcMz4tLHlpio6Y6CraoGITAa+B3yBGaoaIyJTgShVXQQ8KCJDgQLgMDChAms23mL3/2DerVCjLkz8FmrUd6wPauR4pucZZOcV8Oq325j16z5aB9dk4X2X0jO0XgUXbUzlJaru6fmIiIjQqKgot7y3cZNdy2HnEsf3Bbmw7kMIDodxn0LtZmf1UlF7D/Pogo3sPZTNxH5hPH5NB2r4261ujfcTkWhVjShrm80UNRdGRhLMHQMo+Po71rW+Eka857hCd9L6/Ud4b+UeFm9JJqRuDebe3ZdL2jQ4847GVAEW6ObCWP4iaCFMjoR6YU7vpqrsSctiVVwaX21MInLvEWpV9+PeK9rwwIC2BAXYj7AxJ9hvg6l4B7bA+o/gkgecDvPjeYXM+HkPH6/ZT+LR4wCENQjk6Rs6ckuvFhbkxpTBfitMxVv6FFSvA5c/esamBYVFLIxO4I1lOziYkctl4cHce2UbLmsbTMsGgTam3JjfYYFuXE/V8QWw60fH1zUvQ43yR6AcSM9hfmQ88yL3k5yeQ4/QuvxrTE96tzrzaBdjjIMFunGt/ByYOQSS1p9cV6+V4+HMZcjJL+SlxbF8tGY/hUXKZeHBTB3WmUEXNbKrcWPOkgW6ca217zrCvO/9UL149Ernm8DP/7Sm8Yezuf+jdWxOTGd835bcdVkrWjZw7k6KxpjTWaAb18k6BD/9HcKvhiEv/27TpVsP8uiCjRSpMn38xVzdqckFKtIY72WBblznp9cg7xgMnlpuk4ycfKZ+tZWF0Ql0alabt2/taVflxriIBbpxjUO7IPI96HkbNLrotM3ZeQX8EJvCS4tjSTmWy+QBbXnwqnC754oxLmSBblxj2bPgGwBX/t9vq4qKlO9iDvDF+kR+2plKTn4RbRrW5NP7LqV7C+dnhxpjnGOBbs7f4d2Oe5df8Weo1RiAn+PSeOXbbWxOTKdpneqMimjBNZ2a0LtVffx87arcmIpggW7O38b5gEDP24ned4Q3l+1g5c40QurW4B+3dGN49xB8fGwIojEVzQLdnB9V2DiXjGaX8sDCBFbuTKN+TX+evO4ixl/SkurV7A6IxlwoFujmvKRtXUHw0X08k3IdW2tk8JdrOzCub0tq2r1WjLng7LfOnJOCwiLe/Wk3jZb/k+skgNaXj+HFgZ0I9LcfKWPcxX77zFmLP5zNQ/PWE7M/hQ2BayhqP4w/Dunm7rKMqfIs0I1TVJXUY7n8tDON5xbFAPDRZWnUiMyC3uPdXJ0xBizQze8oLFI+X5/IzJ/3sDs1i+P5hQD0DK3LW6N70GLxbVCnBbTs7+ZKjTHgZKCLyBDgLRwPiX5PVV8pp91IYAHQS1XtgaEeSlVZsSOVV7/dxrYDx+jUrDZj+4QSWj+QVsE1ubRNA/xiv4BdP0D/R8DHxpUbUxmcMdBFxBeYBgwGEoBIEVmkqltLtasFPAisqYhCzYVxKDOXp77cwuLNB2jZIJB/j+3BdZ2bnjqOfM278O2fIfQS6Peg+4o1xpzCmSv03kCcqu4GEJF5wDBga6l2zwOvAWd+LI2plL7bcoAnP9/MsZwCHrumPXdf1hp/H4Wd30NepqNRQiSs+Q90uMHxgOdqNdxbtDHmN84EeggQX2I5AehTsoGI9ABaqOrXIlJuoIvIJGASQGho6NlXaypEVm4BzyyK+e0OiB/f3Z32TWo5Hlax4C6I/erUHS6eANf/A3xs0pAxlYkzgV7WnG39baOID/AGMOFML6Sq04HpABEREXqG5uYC2JKYzh/nrmfvoSwmD2jLQ4PCqebrA8ePwryxsO9nuPoFaDfEsYNfANS1/xkbUxk5E+gJQIsSy82BpBLLtYDOwIriR4Y1ARaJyFD7YLRym7d2P099uYXgoADm3t2Xvq0bODZkHYJZf4C0HTDifegy0r2FGmOc4kygRwLhItIKSARGA2NPbFTVdCD4xLKIrAAetTCvvFSVf/0Yxz+W7uDydg355+ju1A0s8Yi45S9A2na4dSG0GeC+Qo0xZ+WMga6qBSIyGfgex7DFGaoaIyJTgShVXVTRRRrXKSxSnlm0hTmr93NTzxBeHdHV0cVyQup2iJ4Fve60MDfGwzg1Dl1VFwOLS617upy2V55/WaYiJBzJ5vGFm/hl1yHuuaI1TwzpQHE32UlLnwH/mo57mxtjPIrNFK0CVJVPouJ5/utYVJVXR3RhVK8yPtjc8xPs+BYGPQs1g0/fboyp1CzQvZiq8suuQ7y5bAeRe4/Qt3V9/jayGy3qVIPC/FKNi2DJX6F2c+hzr3sKNsacFwt0L7Ux/ijPf72VqH1HaFK7Oi/e2JkxPZvis/gRWD+7/B1vnG6ThYzxUBboXig2OYNx760hMMCX54d14uaIFlTXHJg/FuKWQs/boW6L03es3Ry63HzhCzbGuIQFupdJOnqciTMjqe9fyKdDqxEclAgJ+2HZc5C0Dm54EyImurtMY0wFsED3IunH85kwcy1Zufn82PQfBC6MPLnRNwBumQ0X3eC+Ao0xFcoC3UscyszlzllR7EnL4uuBaQSujIQr/w9C+zoa1AuDei3dWqMxpmJZoHuBuJRjTPwgkpSMXKaN6kz75X+ARp3g8kftBlrGVCEW6B7ul11p3Ds7Gn8/H+ZN6kuPxLlwZC+M+9TC3Jgqxh4148EWRMVz2/traVy7Op/f348eDYH/vQptBkLbQe4uzxhzgdkVuqcpzEeXv8yumEjqpmUxv7Y/XRvVodr3PpCRCDnpMPh5d1dpjHEDC3RPkpuJLrgdiVtGflELOtWsTtM61ZHMzJNtBj8HTTq7r0ZjjNtYoHuKrDT46GY0aQNP5N9Nkysn8fCg8NNvrmWMqbIs0D3BkX0w5yaKjsbzQMEj5LcbwisW5saYUizQK7sDm2HOCLQgh4eqPcMGvw4sHtnNwtwYcxob5VKZ7VkJM69Dffx4ucmbfJMexluje1Cvpv+Z9zXGVDkW6JVV7jH4eBTUasr77f/D9G0BPDK4Hb1b1Xd3ZcaYSsoCvbJKjIb8LP7XegovrDzGiJ7NeWBAW3dXZYypxCzQK6t4x421Hlrlx2Xhwbwyoov1mxtjfpdTgS4iQ0Rku4jEicgTZWy/V0Q2i8gGEVklIh1dX2rVUVSkxG9eQZyG0KxxE94Zd/GpD3I2xpgynDElRMQXmAZcC3QExpQR2B+rahdV7Q68BvzD5ZVWEfGHsxn7318JSt1AUlAXPryzN0EBNhjJGHNmziRFbyBOVXcDiMg8YBiw9UQDVc0o0b4moK4ssqpYu+cwd3wQSUuSqCeZXDbwOiQowN1lGWM8hDN/x4cA8SWWE4rXnUJEHhCRXTiu0B8s64VEZJKIRIlIVGpq6rnU67XW7D7EhJlraVw7gA+vdvSVS4vebq7KGONJnAn0sj6JO+0KXFWnqWob4M/AX8t6IVWdrqoRqhrRsGHDs6vUi63efYgJMyNpWqc6cyf1pcGRjZHEfMQAAA1hSURBVBBQG4Lbu7s0Y4wHcSbQE4CSTxRuDiT9Tvt5wPDzKaoq2RB/lIkzIwmpV4N5ky6hUa3qjhEuIReDj30QaoxxnjOJEQmEi0grEfEHRgOLSjYQkfASi9cDO11XovdKOJLNXbOiCK7lz9y7+9KwVoBjQlFKDFh3izHmLJ3xQ1FVLRCRycD3gC8wQ1VjRGQqEKWqi4DJIjIIyAeOALdXZNHeICMnnzs/iCK3oJB5k/o4whwgaT1oETTv5d4CjTEex6nxcKq6GFhcat3TJb5/yMV1ebWCwiImf7yeXamZzLqjN20b1Tq5MX6t49/mEe4pzhjjsWyAsxu8uDiWn3ak8vJNXejXNvjUjQlRENwOatRzT3HGGI9lgX6hqELiOpZv3s22X/cwtUsTxgTvgd17Tm2XsBbaXeueGo0xHs0C/UL5dRoseZIBwAB/HB8bl/fRcctLL1xdxhivYYF+IWQfpmjFq0RKVz6uPpoXh3cufzq/rz+E9Lyw9RljvIIF+gVw9LsXqJWXyatM4LWJowhqFOTukowxXshmrlSwuNiNBG36gC9lIC/dezNtLcyNMRXEAr0CbU5IZ8/8x8ijGhff/hodmtR2d0nGGC9mXS6utnMZRL1Pbl4ex/YeYjDrSe/zKC3D2ri7MmOMl7NAd6V1s+Grh9CgxsQfD6R2UREZba6nzsAp7q7MGFMFWKC7giqsfB1+fAFtcxV/8XuUeRuP8J9xPencuam7qzPGVBEW6Odq1Ruw6k3H96qQmw5dR/FB8KPMW7yTKYPaMcTC3BhzAVmgn4tDu+DHF6B5b2jSxbGuQVuiG9/Ei9PXMrhjY/44sK17azTGVDkW6Odi2TPgVx1u/gBqNQbgUGYuD/xzFc3q1uD1m7vh41PWc0GMMabi2LDFs7XvV4j9Cvo9/FuYFxYpD8/fwOHsPN6+tSd1alRzc5HGmKrIAv1sqMKSJ6FWU7jkgd9Wv7F0Byt3pvHc0E50DqnjxgKNMVWZdbmcjS2fQmI0DHsb/AMB+DQ6gX8vj+OWiOaM7tXiDC9gjDEVx67QnRX3Ayx6EJp0hW6jAViz+xBPfLaJS1o34IXhXRCxfnNjjPtYoDtj0yfw8S1QvzXcugB8fNmblsU9c6JpUT+Q/4y7GH8/+09pjHEv63IpTRWi3ocDmx3LuZmwZSGEXQajP4LqdcjOK+DuD6MQYMbtvagTaB+CGmPcz6lAF5EhwFs4HhL9nqq+Umr7I8BdQAGQCtyhqvtcXGvFKyyArx+G9bMhsAH4FP/n6X4rXP8PqFYdVeWvn28hLjWT2Xf0ISy4pntrNsaYYmcMdBHxBaYBg4EEIFJEFqnq1hLN1gMRqpotIvcBrwGjKqLgCpOXDZ/eCdsXw+WPwYAnoYw+8XmR8Xy2PpEpg9rRPzy4jBcyxhj3cKbjtzcQp6q7VTUPmAcMK9lAVZeranbx4mqguWvLrGDZh2H2jbD9W7judRj41zLDfEtiOs8siuGy8GCbCWqMqXSc6XIJAeJLLCcAfX6n/Z3At2VtEJFJwCSA0NBQJ0usYOkJMGcEHN7tmPnZaXiZzbYkpjNh5loa1PTnzVHdbSaoMabSceYKvazk0jIbiowDIoC/lbVdVaeraoSqRjRs2ND5KitKyjZ4/2rISIJxn5Ub5qt2pjHq3V8J8PNlzl19aBAUcIELNcaYM3PmCj0BKDljpjmQVLqRiAwCngSuUNVc15RXgfavcQxF9AuAiYtP3mSrlC83JPLogo20aRjErDt607h29QtcqDHGOMeZQI8EwkWkFZAIjAbGlmwgIj2Ad4Ehqpri8ipdbdtiWDgRaofA+M+gXthpTQqLlNeXbOedFbvo06o+02+LsHu0GGMqtTMGuqoWiMhk4HscwxZnqGqMiEwFolR1EY4uliBgQfFsyf2qOrQC6z47iesg56jj+4MxsPRpaNrdMUmo5ukjVTJy8nl43gZ+3JbCmN6hPDe0k00cMsZUek6NQ1fVxcDiUuueLvH9IBfX5RpFRbD0Kfj136eubzsIbp4FAUGn7RK97wiPLthI/OFsnh/emXF9Qm1KvzHGI3jvTNGCPPjyAdj8CfS6G7qMdKz3qQZNu4HvqYeek1/IG8t28N+fdtO0Tg0+vrsvvVvVd0Phxhhzbrwn0LMPO2Z45uc4lveudHxd9TT0f6TMceUnbE5I508LNrDjYCZjerfgyes7EhTgPf9pjDFVg3ek1tF4mHMTpO04uc6vBgz9N/QcX+5ueQVF/Ht5HNOWxxEc5M/Mib0Y0L7RBSjYGGNcz/MD/eBWx8SgvEy4/Wto2e/kNp/yP8jcnJDOnz/dxNbkDG7qEcIzf+hkN9kyxng0zw70lFiYOcRxNT7xW2jS+Yy7ZOUW8I+lO5j58x4aBAXw7viLuaZTkwtQrDHGVCzPDvTVbzvukHjPEqjXstxmhUVK1N7DLNl6kK83JXEwI5db+4Ty+JAONrbcGOM1PDfQ849DzBfQcVi5Yb7/UDYfrd3Hp9EJpGXm4e/rQ//wYP49tg29wmwEizHGu3huoG/7BnIzoPuY31apKnvSslgVl8bSrQdZuTMNXx/hqg6NGNq9GVe2b2SjV4wxXstz023jXKjdHFr2B+D7mAO88M1W4g8fByC0fiBTBrVjVK8WNKlj918xxng/zwz0jGTY9SP0n0K+wt8WxzL9p910bFqb54e34fLwYFo2sCcJGWOqFs8M9M0LQIs41PYm7vvvGtbuPcy4vqE8dUNHAvx83V2dMca4hecFuipsnMvxRj0YOjeFQ1m5vDmqO8N7hLi7MmOMcSvPC/QDmyBlK6/rXeT7F7Hgnkvp0ryOu6syxhi387hA37p8Lm3Uj411BvLFHf1oVreGu0syxphKweMCPfuSx3jpeC8+GHeNDUE0xpgSPC4RI1o1IOLOEe4uwxhjKh17DI8xxngJC3RjjPESFujGGOMlnAp0ERkiIttFJE5Enihj++Uisk5ECkRkpOvLNMYYcyZnDHQR8QWmAdcCHYExItKxVLP9wATgY1cXaIwxxjnOjHLpDcSp6m4AEZkHDAO2nmigqnuLtxVVQI3GGGOc4EyXSwgQX2I5oXjdWRORSSISJSJRqamp5/ISxhhjyuFMoEsZ6/Rc3kxVp6tqhKpGNGzY8FxewhhjTDmc6XJJAFqUWG4OJJ3vG0dHR6eJyL5z3D0YSDvfGjxQVTzuqnjMUDWPuyoeM5z9cZf7vE1nAj0SCBeRVkAiMBoYexZvXiZVPedLdBGJUtWI863B01TF466KxwxV87ir4jGDa4/7jF0uqloATAa+B2KBT1Q1RkSmisjQ4oJ6iUgCcDPwrojEuKI4Y4wxznPqXi6quhhYXGrd0yW+j8TRFWOMMcZNPHWm6HR3F+AmVfG4q+IxQ9U87qp4zODC4xbVcxqwYowxppLx1Ct0Y4wxpVigG2OMl/C4QD/TjcK8gYi0EJHlIhIrIjEi8lDx+voislREdhb/W8/dtbqaiPiKyHoR+bp4uZWIrCk+5vki4u/uGl1NROqKyEIR2VZ8zi+pIud6SvHP9xYRmSsi1b3tfIvIDBFJEZEtJdaVeW7F4Z/F2bZJRHqe7ft5VKA7eaMwb1AA/ElVLwL6Ag8UH+cTwA+qGg78ULzsbR7CMTz2hFeBN4qP+Qhwp1uqqlhvAd+pagegG47j9+pzLSIhwINAhKp2BnxxzHHxtvP9ATCk1Lryzu21QHjx1yTgnbN9M48KdErcKExV84ATNwrzKqqarKrrir8/huMXPATHsc4qbjYLGO6eCiuGiDQHrgfeK14WYCCwsLiJNx5zbeBy4H0AVc1T1aN4+bku5gfUEBE/IBBIxsvOt6r+BBwutbq8czsM+FAdVgN1RaTp2byfpwW6y24U5ilEJAzoAawBGqtqMjhCH2jkvsoqxJvA48CJu3Y2AI4WT24D7zzfrYFUYGZxV9N7IlITLz/XqpoIvI7j1tvJQDoQjfefbyj/3J53vnlaoLvsRmGeQESCgE+Bh1U1w931VCQRuQFIUdXokqvLaOpt59sP6Am8o6o9gCy8rHulLMX9xsOAVkAzoCaOLofSvO18/57z/nn3tECvkBuFVUYiUg1HmH+kqp8Vrz544k+w4n9T3FVfBegHDBWRvTi60gbiuGKvW/wnOXjn+U4AElR1TfHyQhwB783nGmAQsEdVU1U1H/gMuBTvP99Q/rk973zztED/7UZhxZ9+jwYWubkmlyvuO34fiFXVf5TYtAi4vfj724EvL3RtFUVV/6KqzVU1DMd5/VFVbwWWAycea+hVxwygqgeAeBFpX7zqKhwPj/Hac11sP9BXRAKLf95PHLdXn+9i5Z3bRcBtxaNd+gLpJ7pmnKaqHvUFXAfsAHYBT7q7ngo6xv44/tTaBGwo/roOR5/yD8DO4n/ru7vWCjr+K4Gvi79vDawF4oAFQIC766uA4+0ORBWf7y+AelXhXAPPAduALcBsIMDbzjcwF8dnBPk4rsDvLO/c4uhymVacbZtxjAA6q/ezqf/GGOMlPK3LxRhjTDks0I0xxktYoBtjjJewQDfGGC9hgW6MMV7CAt0YY7yEBboxxniJ/wdBYGY2jQ345wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores)\n",
    "plt.plot(test_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sigmoid + Sigmoid + Cross Entropy Cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net2(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net2, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(28*28, 30)\n",
    "        self.fc2 = nn.Linear(30, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.logsigmoid(self.fc1(x))\n",
    "        x = F.logsigmoid(self.fc2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net2 = Net2()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net2.parameters(), lr = 1e-1)\n",
    "loss_scores2, train_scores2, test_scores2 = fit(net2, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXhV1b3/8fc3M5kHAmQAEgbDEObIIM5UBSyC0/2Vtmodyo/qbW1rq7e2vdpW294OthVr1apV77VaxXmuoqIoKgGZQphnCEmAQMIQMq37xzlyqRII5CT7DJ/X85wnOeesnP3drPhxZ+211zbnHCIiEvqivC5AREQCQ4EuIhImFOgiImFCgS4iEiYU6CIiYSLGqw137drVFRQUeLV5EZGQtHDhwp3OueyjvedZoBcUFFBaWurV5kVEQpKZbWrtPQ25iIiECQW6iEiYUKCLiIQJBbqISJhQoIuIhAkFuohImFCgi4iEiZAL9Iq9B/nVa+Vs3Lnf61JERIKKZxcWnawFG2t48P0N3D93Paf368r00b340qBuxMdEe12aiIinQi7QLxqWy+iCTJ4q3cKTn2zmhr8vIq1LLF8emsMlI/MY2SsDM/O6TBGRTmde3bGopKTEtffS/+YWx/trqnnu0228UbaD+sYW8tK7MKm4B5OH5jA8P52oKIW7iIQPM1vonCs56nuhHOhH2neoiTeW7+CVZRW8v6aaxmZH99R4vjSwO18a1J1xfbJIiNWwjIiEtogI9CPtPdjIWysqeau8krmrqznQ0EyX2GjG9c3i7KJszuyfTe+sRA3NiEjIOVagh9wYelukdYnl0lH5XDoqn/rGZuav38W7K6t4d3U1b6+sAiA/owun9+vKuL5ZjOuTRbfUBI+rFhFpn7A8Qj+WDTv3M29NNfPW7uTDdbuoq28CoE92EmP7ZDG6IJNTCzPJS+/S6bWJiBxPxA25tFVTcwsrKmr5aP0u5q/bRenGGuoO+QI+L70LI3tnUNI7g5G9MhiQk0JsdMhN2xeRMKNAb6PmFsfKHbV8smE3pZtqWLixhh219QAkxEYxNC+d4b3SGd4znWE908lNS9A4vIh0KgX6SXLOsW3PQT7dvMf32FJD2bZaGppbAOiaHM/Q/LTDj+K8NLqlaCxeRDpOxJ0UDRQzIz8jkfyMRKYMywXgUFMzq3bUsWTLHhZv2cvSrXt4Z1UVn/1/sXtqPEPyfOFenOv72j01XkfyItLhFOgnKD4mmqH56QzNT+eKcb7X9h9qYvm2vSzbtvfw1zkr/y/ks5LiGJSbyuDcNAbnpjI4N5WCrCRd9CQiAaVAD4Ck+BjG9MliTJ+sw6/tP9REeUUty7ftpWx7LWXba3lo3noam30pnxQXzYCcVAblpDIo1/e1qEeKLn4SkZOmMfROdKipmTWV+1ixvZay7Xspr6hjRUUt+/wza6IM+mYnMzAnlYFHBH12SrzHlYtIsNAYepCIj4n2ja3npQE9AWhpcWypOUB5Ra0/6Gsp3bibF5dsP/xzXZPjGZiTcjjgB+WkUtg1iRhNoxSRIyjQPRYVZfTOSqJ3VhITi3MOv77nQAMrKmopr6g7HPYPz9tweMgmPiaKAT1S/uVIfkBOKsnx6lKRSKUhlxDS0NTCumrfkE15RS0r/I89BxoBMIOCrKT/G5f3n4DVVEqR8KEhlzARFxN1eHz9M845dtTWHx6uWbG9lmXb9vLKsorDbbomxzM4N5XiPN9Mm+LcNHpmdtFUSpEwo0APcWZGTloXctK6MGFg98Ov19Y3Uu4P+TL/SdgP1u6kqcX3F1lKQox/nnwqxXlpDMlL01RKkRCnQA9TqQmxX5hKWd/YzOrKOpZv8wX88u21PDp/Ew1Nvitfk+NjGJybytD8NIbkpzM0L03LDIuEEAV6BEmI/b+Loj7T2NzCmsp9hy+IWrZtrz/kNwCQmhDD0Px0hvVMY1i+bx0bLTUsEpx0UlS+oLG5hdWVdSzdupelW/eyZMseVlXW0ewfrslJS2B4T1+4j+iVwZC8NLrE6YIokc6gk6JyQmKjo/zLFKQxfbTvtfrGZsq217J4yx7/OjZ7eG35DgCio4yBOSmM6JnByN7pjOqVqZOuIh7QEbqctF37DrF4i28lykWba1iyZQ/7G5oB38yakt4ZlBRkUFKQyeDcVK0nLxIA7TpCN7OewGNAD6AFeMA596fPtfkacIv/6T7gW865Je2qWoJeVnI8EwZ2Pzy7prnFsWpHHQs317BoUw2lm3bzepnvKL5LbDTDe6ZzamEmYwozGdErncQ4/YEoEkjHPUI3sxwgxzm3yMxSgIXANOfciiPanAaUO+dqzGwScLtzbsyxPldH6JGhsrae0o01LNi4mwUbd1NeUUuLg5goY0h+GmP7ZDGmMJOSgkxd5SrSBgG9wYWZvQDc45x7s5X3M4Dlzrm8Y32OAj0y1dU3snBTDR9v2M0nG3azZMsemloc0VHGsPw0xvXN4rS+XRnVO0MrT4ocRcAC3cwKgPeAYudcbSttfgAMcM5dd5T3ZgAzAHr16jVq06ZNbd62hKcDDU0s3FTD/HW7mL9+F0u37qW5xREXE8WpBRmM79eVM/plMzg3VRc9iRCgQDezZGAucKdz7tlW2pwD3Auc7pzbdazP0xG6HE1dfSMLNu7mg7W7+GDtTlbuqAMgMymO8f26cmb/rpx1SrbmwkvEave0RTOLBZ4BHj9GmA8FHgQmHS/MRVqTkhDLuQO6c+4A34nW6rpDfLB2J++tqeb9NTt5yb+s8MCcVM4uyuacom6M7JWupYRFaNtJUQMeBXY7577bSptewNvAlc65D9uyYR2hy4lyzlFeUcfc1dW8u6qKhZtqaGpxpCbEcFZRNyYM6MbZRdmkJ8Z5XapIh2nXkIuZnQ68DyzDN20R4FagF4Bz7j4zexC4FPhsULyptQ1+RoEu7VVb38gHa3YyZ2UV766qYue+BqKjjJLeGZw3qDvnD+pBr6xEr8sUCaiAznIJFAW6BFJLi2PJ1j28VV7JWyuqWFXpG3sf0COFicU9mFScwyndk3X1qoQ8BbpEnE279vPmikreKNtB6aYanIM+XZOYPCSHyUNyGJiTonCXkKRAl4hWVVfPP8sqeW15BfPX7aLFH+5fHpbLRcNy6NctxesSRdpMgS7it2vfId4oq+TlpduZv34XzvmGZaaNyOOiYbnkpnfxukSRY1KgixxFVW09ry6r4IUl2/l08x7MYExhJpeOzGfSkBwtRSBBSYEuchwbd+7nhcXbee7TrWzcdYAusdFMKu7B5SU9GdsnU+PtEjQU6CJt5Jxj0eYaZi/cxstLtlN3qIneWYlcPiqfy0t60l1XqIrHFOgiJ+FgQzOvl1XwjwVb+Gj9bqKjjAkDujF9TC/O6p+ttWXEEwp0kXbasHM/Ty7YzOzSreza30CvzES+PrYXl4/qSUaSrkyVzqNAFwmQhqYWXi/bwf98tIlPNuwmPiaKacPz+Mb4AgbmpHpdnkQABbpIB1i1o45H52/k2UVbqW9sYWyfTK47vQ/nDuim4RjpMAp0kQ6050AD/1iwhUc/3Mj2vfX06ZrENacXctmofN2kQwJOgS7SCRqbW3ht+Q4efH89S7fupWtyHFePL+TrY3uT1iXW6/IkTCjQRTqRc46P1u/mvrnrmLu6muT4GK4Y15vrTi8kKzne6/IkxCnQRTyyfNte/jJ3Ha8uqyAhJpqvjenFjLP60C1F89nl5CjQRTy2tmof976zlheWbCc22rhyXAH//8w+OmKXE6ZAFwkSG3buZ9acNTy/eBsJsdFcM76QGWf1ITVBY+zSNgp0kSCztmoff3xrNS8vrSA9MZZvndWXq04r0KwYOS4FukiQWr5tL799YxVzV1eTm5bADy4oYtrwPM1jl1YdK9B1q3QRDxXnpfHoNaN54ptjyUqO5/tPLWHKPfP4cN1Or0uTEKRAFwkC4/pm8cIN4/nTV4az50AjX/3rx3zrfxayZfcBr0uTEKJAFwkSUVHG1OF5zLnpLG467xTeXVXNhLvm8vt/ruJgQ7PX5UkIUKCLBJmE2Gi+PaE/b//gLCYV92DW22s57w9zeXNFpdelSZBToIsEqZy0LvzpKyN4csZYEuOi+eZjpVz36AK27TnodWkSpBToIkFubJ8sXvnOGdw6eQAfrN3FeXfN5aF5G2hu8WaGmgQvBbpICIiNjmLGmX355/fOZExhJr94eQUX3/sBq3bUeV2aBBEFukgI6ZmZyMPfOJVZ00ewreYgX571PnfPWUNjc4vXpUkQUKCLhBgzY8qwXN78/llMLM7hrjdXM/UeHa2LAl0kZGUmxTFr+gjuv2IUVXX1TJk1jwfeW6ex9QimQBcJcRcM7sEb3z2Ts4uy+eWrK5n+1480EyZCKdBFwkBWcjz3XzGK310+jLJte5n0x/d4dVmF12VJJ1Ogi4QJM+OyUfm8euMZFGYnc/3ji7hl9lJdZRpBFOgiYaZ3VhKzZ47j+rP78tTCLUz98zzWVumEaSQ4bqCbWU8ze8fMys2szMxuPEobM7O7zWytmS01s5EdU66ItEVsdBQ3TxzAo1ePZue+Bi665wOe+3Sr12VJB2vLEXoTcJNzbiAwFrjBzAZ9rs0koL//MQP4S0CrFJGTcuYp2bz6nTMozk3je/9Ywk+fX05Dk+ash6vjBrpzrsI5t8j/fR1QDuR9rtlU4DHn8xGQbmY5Aa9WRE5Yj7QE/v7NMcw4sw///dEmvvLAfCpr670uSzrACY2hm1kBMAL4+HNv5QFbjni+lS+GPmY2w8xKzay0urr6xCoVkZMWEx3FrZMHcs9XR7ByRx0X3j2PhZt2e12WBFibA93MkoFngO8652o///ZRfuQLVzc45x5wzpU450qys7NPrFIRabcvD83l+RvGkxQfzfQHPubp0i3H/yEJGW0KdDOLxRfmjzvnnj1Kk61AzyOe5wPb21+eiATaKd1TeOGG8ZxamMEPZy/ljpdX6OrSMNGWWS4GPASUO+fuaqXZi8CV/tkuY4G9zjld1SASpNIT43jk6tFcNa43D87bwIzHStl/qMnrsqSd2nKEPh64AjjXzBb7H5PNbKaZzfS3eRVYD6wF/gpc3zHlikigxEZH8bOpxfxi6mDeWVXFv92vk6Whzpzz5k+tkpISV1pa6sm2ReRfvbOyin//+yJSu8Tyt6tPZUCPVK9LklaY2ULnXMnR3tOVoiLCOQO68dTMcbQ4x+X3zeej9bu8LklOggJdRAAYnJvGM986jW4p8Vz58Ce8psW9Qo4CXUQOy89IZPbM0yjOTeX6vy/i8Y83eV2SnAAFuoj8i4ykOB6/biznFHXjx88t576567wuSdpIgS4iX9AlLpr7rxjFlGG5/Pq1lfzm9ZV4NYFC2i7G6wJEJDjFRkfxx/83nOT4aO59dx0HGpq5bcogfJemSDBSoItIq6KjjF9ePITEuBgemreBxuYWfjG1mKgohXowUqCLyDGZGT+5cCCx0VHcN9d3E+pfXjxEoR6EFOgiclxmxi0Ti4iNNma9vZbmFsd/XTpUoR5kFOgi0iZmxk3nF2Fm3D1nDTHRUfzy4mKNqQcRBbqInJDvfak/Tc0t3PvuOuKijdsvGqxQDxIKdBE5IWbGDy8oorG5hb++v4G4GN/NMxTq3lOgi8gJMzNunTyQhiZfqKcmxPLtCf29LiviKdBF5KSYGbdNGUzdoSZ+/+ZqkhNiuHp8oddlRTQFuoictKgo4zeXDmVffRM/e2kFKQmxXDYq3+uyIpYu/ReRdomJjmLWV0cwvl8WtzyzlHdWVnldUsRSoItIu8XHRHP/FSUMzEnh+scX8enmGq9LikgKdBEJiOT4GP72jdFkp8RzzSMLWFe9z+uSIo4CXUQCJjslnseuGU2UGVc+9AlVdbpHaWdSoItIQBV0TeLhb5zK7v0NXPdoKQcamrwuKWIo0EUk4Ib1TOfu6SNYtm0vNz65mOYWraXeGRToItIhzhvUndu+PIg3V1RyxysrvC4nImgeuoh0mG+ML2TT7gP87YON9OuWzNfG9Pa6pLCmI3QR6VA/uXAQZxdlc9sLZXy4dqfX5YQ1BbqIdKjoKOPu6SMo7JrEtx5fxIad+70uKWwp0EWkw6UmxPLQVacSZXDtowuorW/0uqSwpEAXkU7RKyuRv3x9FJt3HeB7Ty6mRTNfAk6BLiKdZmyfLP5zyiDmrKziD2+t9rqcsKNAF5FOdcXY3vxbST6z3l7La8sqvC4nrCjQRaRTmRm/mFbMiF7p3PT0EtZU1nldUthQoItIp4uPiea+r48iMS6amf+zkH2HtDxAICjQRcQT3VMTmDV9JBt3HeDm2UtwTidJ2+u4gW5mD5tZlZktb+X9NDN7ycyWmFmZmV0d+DJFJByN65vFLROLeHXZDh58f4PX5YS8thyhPwJMPMb7NwArnHPDgLOB35tZXPtLE5FI8M0z+jCpuAe/fn0ln2zY7XU5Ie24ge6cew841r+yA1LMzIBkf1sNiIlIm5gZv7lsKD0zuvDtJxaxa98hr0sKWYEYQ78HGAhsB5YBNzrnWgLwuSISIVISYvnz10ZSc6CR7z21RBcdnaRABPoFwGIgFxgO3GNmqUdraGYzzKzUzEqrq6sDsGkRCReDc9O4bcog3ltdzb3vrvW6nJAUiEC/GnjW+awFNgADjtbQOfeAc67EOVeSnZ0dgE2LSDj56uheTBmWy11vrubj9bu8LifkBCLQNwMTAMysO1AErA/A54pIhDEzfnXJEHplJvLdfyymZn+D1yWFlLZMW3wCmA8UmdlWM7vWzGaa2Ux/k18Ap5nZMmAOcItzTosei8hJSY6PYdb0kezcd4ibn1mq+ekn4Lh3LHLOTT/O+9uB8wNWkYhEvCH5adwycQB3vFLOf3+0iSvHFXhdUkjQlaIiEpSuPb2Qc4qyueOVcsorar0uJyQo0EUkKJkZv7t8GGldYvnuk4upb2z2uqSgp0AXkaCVlRzP7y4fxqrKOn792kqvywl6CnQRCWpnnZLNNeMLeeTDjbyzssrrcoKaAl1Egt7NE4sY0COFH85ewk4tDdAqBbqIBL2E2Gj+9JUR1NY38R/PLNNUxlYo0EUkJBT1SOHmC4p4q7ySp0u3el1OUFKgi0jIuGZ8IWP7ZPKzl8rYsvuA1+UEHQW6iISMqCjfVMYoM77/1GKatSrjv1Cgi0hIyc9I5PaLBrNgYw0Pvq9lo46kQBeRkHPJyDwuGNyd37+5mjWVdV6XEzQU6CIScsyMOy8eQnJ8DDc9vYTGZt1TBxToIhKiuibHc8e0YpZu3ctf3l3ndTlBQYEuIiFr8pAcLhqWy91z1lC2fa/X5XhOgS4iIe3nUweTkRTHD55eSkNTZA+9KNBFJKSlJ8Zx57RiyitqI37oRYEuIiHv/ME9mDo8l1lvr2HF9shdO12BLiJh4fYpg0lPjOWHsyN31osCXUTCQkZSHHdMK6Zsey33z43MoRcFuoiEjYnFOVw4NIe756yNyAuOFOgiElZunzKYxPhobnlmacSt9aJAF5Gwkp0Sz21TBrFo8x4e/XCj1+V0KgW6iISdacPzOKcom9++sYrNuyJnmV0FuoiEnc/WeomOMm59LnLucKRAF5GwlJvehVsmDWDe2p08s2ib1+V0CgW6iIStr43uRUnvDH7x8gqq68L/5tIKdBEJW1FRxq8vHcLBhmZ+/vIKr8vpcAp0EQlr/bql8O/n9uOlJduZU17pdTkdSoEuImFv5ll9OaV7Mj99fjn7DzV5XU6HUaCLSNiLi4niV5cMpaK2nt//c7XX5XQYBbqIRIRRvTP4+pjePPLhBpZs2eN1OR1CgS4iEeOHE4vomhzPj55dRlMYrsioQBeRiJGaEMvPpw5mRUUtD83b4HU5AXfcQDezh82sysyWH6PN2Wa22MzKzGxuYEsUEQmcCwb34LxB3fnDW6vZsju8lgVoyxH6I8DE1t40s3TgXuAi59xg4PLAlCYiEnhmxs8uGkyUGf/5wvKwWhbguIHunHsP2H2MJl8FnnXObfa3rwpQbSIiHSI3vQs3nV/EO6uqeXXZDq/LCZhAjKGfAmSY2btmttDMrmytoZnNMLNSMyutrq4OwKZFRE7OVeN6U5yXyu0vlVFb3+h1OQERiECPAUYBFwIXAD81s1OO1tA594BzrsQ5V5KdnR2ATYuInJyY6Ch+dfFQdu07xG9eX+l1OQERiEDfCrzunNvvnNsJvAcMC8Dnioh0qCH5aVx1WgGPf7yZTzfXeF1OuwUi0F8AzjCzGDNLBMYA5QH4XBGRDnfT+UV0T0ngx88tD/m56W2ZtvgEMB8oMrOtZnatmc00s5kAzrly4HVgKfAJ8KBzrtUpjiIiwSQ5PobbpgxiRUUtj4T4LetijtfAOTe9DW1+C/w2IBWJiHSyicU9OHdAN+56czWTh+SQm97F65JOiq4UFZGI99nc9BbnuP3FMq/LOWkKdBERoGdmIjdOOIV/rqjkrRWhuW66Al1ExO+6Mwo5pXsyt71YxoGG0Fs3XYEuIuIXGx3FHdOGsG3PQe6es9brck6YAl1E5AijCzO5fFQ+D76/nlU76rwu54Qo0EVEPudHkweSnBDDT55fRktL6CzepUAXEfmczKQ4bp00kAUba5i9aKvX5bSZAl1E5CguG5VPSe8MfvVqOTX7G7wup00U6CIiRxEVZdxxcTG19U38+rXQWLxLgS4i0ooBPVK59vRC/lG6hdKNx7otRHBQoIuIHMONE/qTm+ZbvKsxyBfvUqCLiBxDUnwMt100mFWVdfztg+C+sbQCXUTkOM4f1J0JA7rxx7fWsH3PQa/LaZUCXUTkOMyM2/2Ld/3speBdvEuBLiLSBj0zE/n2uf15o6ySt1cG5+JdCnQRkTb65hl96JudxH++UMbBhmavy/kCBbqISBvFxfgW79pac5B73lnjdTlfoEAXETkB4/pmccmIPB54bz1rq4Jr8S4FuojICbr1woF0iY3mJ88vx7ngWbxLgS4icoK6Jsdz88QBfLR+N88v3uZ1OYcp0EVETsJXR/diWM907nylnL0HGr0uB1Cgi4iclKgo485pxeze38Bv/xkci3cp0EVETlJxXhpXnVbA4x9vZvGWPV6Xo0AXEWmP7593Ct1S4vnxc8to8njxLgW6iEg7pCTEctuUwZRtr+Wx+Zs8rUWBLiLSTpOKe3B2UTZ3vbmaHXvrPatDgS4i0k5mxs8vKqaxuYWfv+zd4l0KdBGRAOiVlch3JvTn1WU7eGdllSc1KNBFRALkm2f0oV+3ZH76wnJPFu9SoIuIBEhcTBR3Titma81BZr3d+Yt3KdBFRAJoTJ8sLhuVzwPvrWd1Zecu3qVAFxEJsFsnDyQlIYYfP7eMlpbOW7zruIFuZg+bWZWZLT9Ou1PNrNnMLgtceSIioSczKY4fTR7Igo01PFW6pdO225Yj9EeAicdqYGbRwH8BbwSgJhGRkHf5qHxGF2byq9dWsnPfoU7Z5nED3Tn3HrD7OM2+DTwDeDNXR0QkyJgZv7x4CAcamrjzlfJO2Wa7x9DNLA+4GLivDW1nmFmpmZVWV1e3d9MiIkGtX7dkvnVWX577dBvz1uzs8O0F4qToH4FbnHPHnXTpnHvAOVfinCvJzs4OwKZFRILb9ef0oyArkR8/v4z6xo6dmx6IQC8BnjSzjcBlwL1mNi0AnysiEvISYqO58+IhbNp1oMPnprc70J1zhc65AudcATAbuN4593y7KxMRCRPj+3XlkpF53D93Pat2dNzc9LZMW3wCmA8UmdlWM7vWzGaa2cwOq0pEJMz85MJBpCTEcGsHzk2POV4D59z0tn6Yc+4b7apGRCRMZSbF8ZMLB3HT00v4+yeb+frY3gHfxnEDXUREAuOSkXnMXV1NZlJch3y+Al1EpJOYGXdPH9Fhn6+1XEREwoQCXUQkTCjQRUTChAJdRCRMKNBFRMKEAl1EJEwo0EVEwoQCXUQkTJhznXe/u3/ZsFk1sOkkf7wr0PGLCwefSNzvSNxniMz9jsR9hhPf797OuaOuP+5ZoLeHmZU650q8rqOzReJ+R+I+Q2TudyTuMwR2vzXkIiISJhToIiJhIlQD/QGvC/BIJO53JO4zROZ+R+I+QwD3OyTH0EVE5ItC9QhdREQ+R4EuIhImQi7QzWyima0ys7Vm9h9e19MRzKynmb1jZuVmVmZmN/pfzzSzN81sjf9rhte1dgQzizazT83sZf/zQjP72L/f/zCzjrndi0fMLN3MZpvZSn+fj4uEvjaz7/l/v5eb2RNmlhCOfW1mD5tZlZktP+K1o/av+dztz7elZjbyRLYVUoFuZtHAn4FJwCBgupkN8raqDtEE3OScGwiMBW7w7+d/AHOcc/2BOf7n4ehGoPyI5/8F/MG/3zXAtZ5U1XH+BLzunBsADMO372Hd12aWB3wHKHHOFQPRwFcIz75+BJj4udda699JQH//YwbwlxPZUEgFOjAaWOucW++cawCeBKZ6XFPAOecqnHOL/N/X4fsPPA/fvj7qb/YoMM2bCjuOmeUDFwIP+p8bcC4w298krPbbzFKBM4GHAJxzDc65PURAX+O7BWYXM4sBEoEKwrCvnXPvAbs/93Jr/TsVeMz5fASkm1lOW7cVaoGeB2w54vlW/2thy8wKgBHAx0B351wF+EIf6OZdZR3mj8DNQIv/eRawxznX5H8ebn3eB6gG/uYfZnrQzJII8752zm0Dfgdsxhfke4GFhHdfH6m1/m1XxoVaoNtRXgvbeZdmlgw8A3zXOVfrdT0dzcy+DFQ55xYe+fJRmoZTn8cAI4G/OOdGAPsJs+GVo/GPGU8FCoFcIAnfcMPnhVNft0W7ft9DLdC3Aj2PeJ4PbPeolg5lZrH4wvxx59yz/pcrP/vzy/+1yqv6Osh44CIz24hvOO1cfEfs6f4/yyH8+nwrsNU597H/+Wx8AR/uff0lYINzrto51wg8C5xGePf1kVrr33ZlXKgF+gKgv/9MeBy+kygvelxTwPnHjR8Cyp1zdx3x1ovAVf7vrwJe6OzaOpJz7kfOuXznXAG+vn3bOfc14B3gMn+zsNpv59wOYIuZFflfmgCsIMz7Gt9Qy1gzS/T/vn+232Hb15/TWv++CHFzC5gAAACzSURBVFzpn+0yFtj72dBMmzjnQuoBTAZWA+uAH3tdTwft4+n4/sxaCiz2PybjG0+eA6zxf830utYO/Dc4G3jZ/30f4BNgLfA0EO91fQHe1+FAqb+/nwcyIqGvgZ8BK4HlwH8D8eHY18AT+M4TNOI7Ar+2tf7FN+TyZ3++LcM3C6jN29Kl/yIiYSLUhlxERKQVCnQRkTChQBcRCRMKdBGRMKFAFxEJEwp0EZEwoUAXEQkT/wtXFWDK5fpTXgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3xUVfrH8c+TTkIJPbRAgGAITSAgCiqgIIqCi40qKIi6oq6uuvaC+1vbqqsrFkAFlSqiRmRBRZEOCZ2EkhBKQoAUIJXUOb8/ZpQQEjLAJJOZPO/XK6/MvffMned64evlzLnnijEGpZRSrs/D2QUopZRyDA10pZRyExroSinlJjTQlVLKTWigK6WUm/By1gc3atTItGnTxlkfr5RSLmnz5s1pxpjGZW1zWqC3adOG6OhoZ328Ukq5JBE5VN427XJRSik3oYGulFJuQgNdKaXchAa6Ukq5CQ10pZRyExroSinlJjTQlVLKTThtHLpSSrm8zGTYOgeKCy7sfZcNgRY9HV6OXYEuIkOA9wBPYKYx5vVS24OB2UCgrc3TxpilDq5VKaWqj5Q98NUIyDwCyIW9t06QcwJdRDyBacAgIAmIEpFIY0xsiWbPAwuNMR+JSDiwFGjj8GqVUqo6OLwB5t4FXr7wwBoI6uLsigD7+tB7A/HGmARjTAEwHxheqo0B6tpe1wOSHVeiUkpVI3uWwhfDwb8hTPyp2oQ52BfoLYDEEstJtnUlvQyMFZEkrFfnD5e1IxGZLCLRIhKdmpp6EeUqpZQTbZ4NC8ZAk3BrmNdv4+yKzmJPoJfVOVT6QaSjgFnGmJbATcCXInLOvo0x040xEcaYiMaNy5wsTCmlqh9j4Pe34IdHoN1AGP8DBDRydlXnsOdL0SSgVYnllpzbpTIRGAJgjFkvIn5AIyDFEUUqpdQlOXkIFo6D1H0XuQMDRXnQbRQM+y94eju0PEexJ9CjgFARCQGOACOB0aXaHAauA2aJSEfAD9A+FaWU8x3bBV/dBkWnoddEOLfzwD4N2kLPCSAXOKKlClUY6MaYIhGZAizHOiTxM2NMjIhMBaKNMZHA34EZIvIY1u6YCcaY0t0ySilVtQ6ugXmjwScA7lkGTcOr7KNzC4rIyisit6CY3II/fheTm19EePO6tG4Y4PDPtGscum1M+dJS614s8ToW6OvY0pRS6hLEfg/f3Gf94nLsNxDYqsK3XKqsvEL+t+sY3209wvqEdMq7rH311s6Mc1agK6WUS4maCT8+AS17wegF4N+gUj4mPiWbNXGp7DmWxe6jmew+lkVBkYXWDf35a/92NKtXiwBfT2p5exHg64m/jxf+Pp40r1erUurRQFdKuTZLMcR8C6dPWpdT91gDvcONcPtn4OPvsI8yxhB7NJMlO47yU8wx9qfmAFDf35uOzeoy/srWDOncjB7BgYgT+to10JVSrqswD76dbO1eKanHeBj6Dng6JuKOZ+axeMsRvtt6hL3Hs/D0EPq0bcD4q9owMKwJLQJrOSXAS9NAV0q5prwMmD8GDq6Gwf+0DikE8PCEWvUvefcWi2Ht/jTmbDjMz7uPU2wx9AgO5NXhnRjatTkNAnwu+TMcTQNdKeUaThyANe9A4WnrcvI2OHkARsyArnde0q6Lii2cyC0gLauAmOQMVselsTY+jfScAur7ezOpXwgjewcT0sjxX2Q6kga6Uqr6O7odvrodCnKgTlPrOm9/GL0Q2l93wbvbmJDOqrhU9h7LYvfRLJIzTp81IqVRbV+u6dCYAWFNGBzeFD9vTwcdSOXSQFdKVW8JK2H+WPCrB5OXQOPLLnpXadn5vLoklu+3JePpIbRrHEDP1vW5rWELGtfxpVFtX0IaB3BZ0zrVok/8QmmgK6Wqr13fwOL7oVGodSx53eYXvIusvEL2Hstiy+GTTPttP7kFRTx6XSgPXNuOWj6uceVtLw10pVT1tOFjWPY0BF8Jo+ba/UVnxulC1u9PY3VcGuv2p3MgLefPbb3bNOBfIzrTvkmdyqraqTTQlVLVizGw4hVY8y6E3Qy3zQTv89+IY4xh04ETzNl4mP/tOkphsSHAx5M+bRtye8+WhAXVIaxZXZrX83PJrhR7aaArpaqP4kL44VHYNgd63gND37YOQyyHMYZlu47x7i/72Hc8m7p+Xoy5ojU3dWlG9+BAvD0vciIuF6WBrpSqHgpy4OsJEPcT9H8Grv1HuTMbGmPYkHCCN5btYVviKUKb1Oat27tyc9fmbtcvfiE00JVSzpGTDofWWLtYMLDuA0jeAjf/ByLu+bOZMYaM04UcSs8lLiWbtfFprIlPIzUrn6C6frx5W1du69kSTw/37Uqxlwa6Uqrqpe6FL0dAZtKZdZ6+cOeX0PFmMk4XsnJvCstjjrE2Pp2M04V/NmsQ4EPf9o24JrQRt3Rr7jJjxKuCBrpSqmod3ghz7wRPHxj3HcUBTdh86CQrk2D3Bh8OL13JofRciiyGxnV8GdIpiNCmtWnVwJ+QRgG0b1wbD70aL5MGulKqcu1dBgdWWV9bCmHLl1C3GaduW8CcfR7M23SYpJOn8ffxpHXDAEKb1OHGzs0YENaE7q0CNbwvgAa6UqryrPsv/PQ8eNUCD2vcFDfvwazmL/Hu9ANk5xdxVbuGPH1jGIPDg/DxqlmjUhxNA10p5XgWC/z8Aqz/AMJvhRHTKRJvFkYn8Z9f9pGyL41B4U158obL6NDUPW/ycQYNdKXUxcnLgKVPQeuroOf4M+uLCiByCuxYAL0nY254jeW703hz+R4SUnPo2bo+H47pQUSbynmKUE1mV6CLyBDgPawPiZ5pjHm91PZ3gQG2RX+giTEm0JGFKqWqkcyjMOd2OL4LdsyHjCQY8Kx1LPnCcbD/Vxj4Aid7PMwjszazOi6Ndo0DmD6uJ4PCm7r13ZrOVGGgi4gnMA0YBCQBUSISaXswNADGmMdKtH8Y6F4JtSqlqoO0OOuQw9MnYMw3EPstrHoTMpMhJQaO7oBhH7C3+a3c9+E6jmXkMXV4J0b3Dsarht25WdXsuULvDcQbYxIARGQ+MByILaf9KOAlx5SnlKpWck/A5zdaX09YAs27W+cjr9MMVr0FXrUovmsO3+d24fkP11Lb14v59/ehR/ClP0FIVcyeQG8BJJZYTgKuKKuhiLQGQoBfy9k+GZgMEBwcfEGFKqWqgVVvQW463L8KgrpY14nAwOfJa9yVn45481akL4kntnN5q0A+GdeTpnX9nFtzDWJPoJfV2WXKWAcwElhkjCkua6MxZjowHSAiIqK8fSilqqP0/bBpBnQfdybMsc43PnvdQWau8eNUbiERrf147qZwBoU31dvxq5g9gZ4EtCqx3BJILqftSOChSy1KKVUNrXjFenfngGcBKCy2MGN1Ap/8nkDG6UIGhjXhoQHt6dlau1ecxZ5AjwJCRSQEOII1tEeXbiQilwH1gfUOrVAp5XyJmyD2e+j/LNQJYs+xTP6+cDsxyZlcF9aER68PpWtLHdjmbBUGujGmSESmAMuxDlv8zBgTIyJTgWhjTKSt6ShgvjFGu1KUcieWYlj+HNQO4kj4RBb+vI8PV8ZTr5Y3n4zryQ2dgpxdobIRZ+VvRESEiY6OdspnK6XsVHia4kUT8dz7I+8EPMb76b0AGNqlGa/e2pkGAT5OLrDmEZHNxpiIsrbpnaJKqbKdPkn+l3fhnbyJlwvvZkfAIJ7pHcTgTkGENApwdnWqDBroSqlz5WWQ/fFgfDP285R5lOtHPcjLnbVrpbrTQFdKncViMWz56gV6nIrjhTqvMGXCJNroFblL0PtwlVJ/yskv4oXZS+mSOJfNgYN54dEpGuYuRANdKQVAQmo2t320jl4J0/D09CDinrf18W4uRgNdqRrOGMPC6ERu/u8a6mfEcKvnWrz6TkECW1X8ZlWtaB+6UjVYdn4R//hmBz/uOEqfkPrM8oyEE42g79+cXZq6CBroStVQqVn5TPh8E3uOZfHk4A48aObjsXod3PRv8Kvr7PLURdBAV6oGOpiWw92fbSI1K59Px11O/7jXYMsX0H0sRNzr7PLURdJAV6qGiU3OZNynG7EYw7x7unL5hsdh3//gmidhwHPW6XCVS9JAV6oG+ePK3MfLg7ljOxCyfDwkRVm7WXrf5+zy1CXSQFeqhjiemcfYTzdSbLGwaFRr2nx/G5w8AHfOhvDhzi5POYAGulLu6tA62LEAjCG/yMLm3Sk8WlDEwLAmNPzudyjIhnHfQpt+zq5UOYgGulLuaOci+PYB8PLD+ASQk1tAz2JDfX9vfI54QEBjGPM1BHV2dqXKgTTQlXI3Gz6GZf+A4KsovmsuDy1OYHnsMT4Y1YOhXZs5uzpVifROUaXchTHwy8vWMA+7GTP2G15dkcyymGM8PzRcw7wG0Ct0pdxBcSFEPgLb50LPe2Do23y+7jCz1h1kUr8QJvYLcXaFqgpooCvl6gpy4OsJEPeT9Zmf1z5F9KGT/GvpbgaHN+XZmzo6u0JVRezqchGRISKyV0TiReTpctrcKSKxIhIjInMdW6ZSqlxLn4T4X+Dmd6H/P0jLKWDK3K20qF+Lf9/ZDQ8PvVGopqjwCl1EPIFpwCAgCYgSkUhjTGyJNqHAM0BfY8xJEWlSWQUrpUowBuJXQKcREHEvxRbD3+Zv42RuAYv/ehV1/bydXaGqQvZcofcG4o0xCcaYAmA+UPouhPuAacaYkwDGmBTHlqmUKlNGImQfg1ZXYLEY/vljLGvi03h1eGc6Na/n7OpUFbMn0FsAiSWWk2zrSuoAdBCRtSKyQUSGlLUjEZksItEiEp2amnpxFSulzkiKAqCwRQRPfL2dz9ce5J6+bbizl85lXhPZ86VoWR1wpoz9hAL9gZbAahHpbIw5ddabjJkOTAeIiIgovQ+l1IVKjMJ41WLy8jx+izvJE4M78NCA9s6uSjmJPYGeBJT8331LILmMNhuMMYXAARHZizXgoxxSpVKqTEWHN7JH2vF7/EleH9GFkb2DnV2SciJ7ulyigFARCRERH2AkEFmqzXfAAAARaYS1CybBkYUqpc6WejIDc3QH6/JC+GhsTw1zVXGgG2OKgCnAcmA3sNAYEyMiU0VkmK3ZciBdRGKB34AnjTHplVW0UjXdkVOnmTp9Lt4U0W/ATdzQKcjZJalqwK4bi4wxS4Glpda9WOK1AR63/SilKtHBtBzGzNzIradjQCC89/XOLklVE3qnqFIuZO+xLNuc5obJbdPgZGuorbd9KCudnEspF7Ej6RR3TV+Ph8CC+66gXto2aNXb2WWpakSv0JVyAfuOZzF25kbq1vJm7qQ+BHumQ9ZRaNnL2aWpakSv0JWq5o5l5DHhs034ensy774+BDf0h6RN1o0a6KoEDXSlqrHMvEImfL6JjNOFzLqnF60a+Fs3JEWDVy0I6uLcAlW1ol0uSlVTqVn5PDRnC/Ep2Xx+T6+z52Y5vB6adwdPnXxLnaFX6EpVQyv3pnDje6vYnnSKd+66nKtDG5/ZmJMOydug7bXOK1BVS3qFrlQ1UmwxvLZ0NzPXHOCypnWYM6kPlwXVObtRwm+AgXbXOaVGVX1poCtVTeQVFvPIvK38FHuccX1a89zQjvh5e57bcP+v4BcILXpUfZGqWtNAV6oayMgt5L4vook6dIKXbwlnQt9yngFqjDXQ2/YHjzLCXtVoGuhKOdmBtBzu/zKag2m5/HdUd27u2rz8ximx1vHn7bW7RZ1LA10pJ1q26yhPfr0DT09h1j29uKp9o/O/Yf+v1t/af67KoIGulBMUFVt4Y9keZqw+QLdWgXw4pgctAmtV/Mb4FdA4DOqVfmiYUhroSlW5rLxCHp63lZV7Uxl/ZWueGxqOj5cdI4gLcuHQOug1qfKLVC5JA12pKpR86jT3zooiLiWb10Z0YdSFPJTi0Doozof2AyuvQOXSNNCVqiLbEk8x+YtoThcUM+ueXlzdygcybU9z9PAqexrcwtNw+qT19d4fwcsPWvetuqKVS9FAV6oKLNqcxLPf7qRpXV++mnQFHZIj4c1HwVJ4ptHlY+CW987czp8UDXPvhNwSD/9qdx1429HXrmokDXSlKlFBkYXX/7eHz9Ye4Kp2DZk2qjv1t34AK6ZCyLXQeYS1Ycoe2PgRZKfAnbOt3SsL77ZetQ98HsTWx962v7MORbkADXSlKsnKvSlMXRJLQmoO9/Rtw3M3huH187Ow8WPocgcM/xC8fM68ofFl8OPjML0/pO+HoM4wZpE+kUjZza7JuURkiIjsFZF4EXm6jO0TRCRVRLbZfvRreFVjHc/MY9LsaCZ8HoUx8PmEXrx0Sye89v9sDfMrHoS/TD87zAEi7oG7voJThyHkapjwo4a5uiAVXqGLiCcwDRgEJAFRIhJpjIkt1XSBMWZKJdSolMuITc5k4uwoTuUW8vSNYdzbN+TMkMRtc8C/EQx+FTzKuZYKGwp/3wO+9cpvo1Q57Oly6Q3EG2MSAERkPjAcKB3oStVov+1JYcrcLdTx8+abB68ivHndMxtzT8C+ZdYx5BXNYV6rfuUWqtyWPZcALYDEEstJtnWl3SYiO0RkkYi0KmtHIjJZRKJFJDo1NfUiylWq+jHGMGNVAhNnR9GmUQDfPdT37DAHiFkMxQXQbaRzilQ1gj2BLmWsM6WWfwDaGGO6Ar8As8vakTFmujEmwhgT0bhx47KaKOVSMvMKeeCrzfzf0t0MDg9i4f1XElTP79yG2+ZBk04Q1LXqi1Q1hj1dLklAySvulkByyQbGmBIDZZkBvHHppSlVvW0+dIK/L9xO4snTPD+0IxP7hSBSxvVPWhwciYbB/4SytivlIPYEehQQKiIhwBFgJDC6ZAMRaWaMOWpbHAbsdmiVSlUj+1OzeXPZHpbHHCeorh/zJ/ehV5sG5b9h+zzrOPIud1RdkapGqjDQjTFFIjIFWA54Ap8ZY2JEZCoQbYyJBB4RkWFAEXACmFCJNSvlFPlFxbz90z4+XXMAPy8PHh/UgYn9QgjwPc9fI4sFti+w3uFZJ6jqilU1kl03FhljlgJLS617scTrZ4BnHFuaUtXH/tRsHpm3lZjkTEb1DubvgzvQqLZvxW88uBoyk2DQK5VfpKrx9E5RpcphjCEuJZulO4/yye8J+Hl7MOPuCAaFN7V/J9vnWceUhw2tvEKVstFAV6oMC6IO89HK/RxMzwVgwGWNeW1E17JHsJQnPxtiI6HL7TqhlqoSGuhKlbJkRzL/+GYn3YMDmXR1WwaHN6VJ3QsI8j/s/gEKc+Dy0RW3VcoBNNCVKmHL4ZM8vnA7vdrU56tJV+Dr5XnxO9s+F+qHQKsrHFegUuehk0UoZZN4IpfJX0QTVNePT8ZFXFqYn0qEA6uh2ygde66qjF6hqxqvqNjC4i1HePvnvRQUWZg/uRcNAnwqfuP57FgAGOh2l0NqVMoeGuiqxsrILWTlvhQ++DWeuJRsugcHMnVYZ9o3qX1pOzYGts+3PiqufhuH1KqUPTTQVY1RUGRh6+GTrIlPY1VcGjuTTmEx0LZRAB+P7cENnYLKvnX/QiWshPQ46PvIpe9LqQugga7cXkpWHq/8EMtve1LILSjG00Po1rIeUwaGcnVoI7q3CsTL00FfJ8X9bH10XP02EH6rY/aplJ000JVb25CQzsPztpKVV8htPVpydWhjrmzXkHq1KpiT/GJsmweRU6BJRxjzDfjVrfg9SjmQBrpyS9n5Rcxae4B3ft5Hm4YBfDmxN2FBlRSwxsDa9+CXlyDkGrhrjoa5cgoNdOU2jDH8vi+VxVuO8FPsMfIKLQzt0ow3bu9K7fNNoHUpLBb46TnY8CF0GgF/+Ri87JjjRalKoIGu3MLBtBye+24na+PTCfT35vaeLflL95b0CA50zBedZSnKh+8ehF3fWB/8fMO/9Dmgyqk00JVLK7YYPlm1n/d+icPH04OpwzsxslfwmQczV5a8TFgwFg78Dte/DH3/pjcQKafTQFcuK6+wmEfmbeWn2OMM6RTEy8M6XdjkWRcrOwW+ug2Ox8CtH+lcLara0EBXLikjt5D7vohm26EUZvU6Sv+QE7A/9uxG3rWg4y3n79NOjILUPfZ/sCmGNe9aQ330AggddHEHoFQl0EBXLudoxmkmfBbF8bRU1raaQeOd62FnOY17T4ab3ip726YZsPRJzn3meQX8G8L4H6BlxIW9T6lKpoGuXEp8ShZ3f7oJ77w01jR9j9qpe+CW96yPeCvt9zcg+jPofT80an9mvTHw6z9h9b/hsptgyGsgFzARl39D8PG/9INRysE00JVryEgi5X9vsGNPEk+JcFPdffhkpcOo+dBhcNnvue5FiPnWOj585BzruuIiWPIobP0KetwNQ98FT/1roNyDXUMBRGSIiOwVkXgRefo87W4XESMi+m9R5TBxx7NI+GwSgbvncLVs52b/WHxq1bF2e5QX5gC1m1hHn+xZAofWQUEuLBhjDfNrnoJb3tcwV26lwj/NIuIJTAMGAUlAlIhEGmNiS7WrAzwCbKyMQlXNkpKZR+T2ZL7bdoT6R9fwpc96FjZ8gOvunYqXPQ9n/sOVD0H0p7DsGfD0gaQoGPo29JpUecUr5ST2XJ70BuKNMQkAIjIfGA6UGlLAq8CbwBMOrVDVKBsT0pm2cj9r4lKxGLi8RW3ea7CYYs9g7vzr1Au/C9PHHwa+AN//1Rrod86G8OGVU7xSTmZPoLcAEkssJwFnPVNLRLoDrYwxS0Sk3EAXkcnAZIDg4OALr1a5rcJiC//5ZR8frtxPUF0//tq/Pbd2b0H7I9/B9/vg9s8v/pb6biPh1CFoOwBaX+nYwpWqRuwJ9LJuf/tznJeIeADvAhMq2pExZjowHSAiIuICx4opd3UoPYdH529jW+Ip7oxoyUu3dCLA1wsKcuCLV6FFBHT6y8V/gIcnDHjWcQUrVU3ZE+hJQKsSyy2B5BLLdYDOwErbnBlBQKSIDDPGRDuqUOV+CostfLrmAP/5ZR/enh58MLo7N3dtbt1YXAg/PArZx6zdJHpbvVIVsifQo4BQEQkBjgAjgT/vdTbGZACN/lgWkZXAExrm6ny2Hj7Js9/uYvfRTAaHN+WV4Z1oVq+WdWN+tvUhEftXWPu/g/s4t1ilXESFgW6MKRKRKcBywBP4zBgTIyJTgWhjTGRlF6ncR0pWHm8u28uizUk0revLx2N7MqRz0JkGOWkw5w44us06rLDneOcVq5SLsWsQrjFmKbC01LoXy2nb/9LLUu6mqNjCrHUH+c8vceQXFfPAte2YMrD92fOUnzwIX46AzCPWh0SE3eS0epVyRXpXhap0u45k8PTiHew6ksm1HRrz0i3htG1c++xGR3fAnNutc4zf/b12syh1ETTQlcMZYziUnsvmQydZtz+db7cm0SDAlw9Gd2dol2bWB04cWAUJK61vsBRB9OfgWwfujYQmYU6tXylXpYGuHCo9O58xMzey51gWAHV8vRjVO5inbgijnr/twcybZ8GSx6yvxTb7RFAXazdLvRZVX7RSbkIDXTnM6YJiJs6O5kBaDq8M60Sftg1p36Q2nh62IYfGwKq34Lf/g/aDrMMRfQKcW7RSbkQDXTlEUbGFh+dtYUfSKT4a25MbOtlGrqx6CxJ+t74uyIbkrdBtNAx7Hzy9nVewUm5In2irLlnG6UKeXryTX3an8MqwTmfC/OBa67zjuelgLOBtm1fl1g81zJWqBHqFri5a4olcPlt7gIVRieQUFPPwwPaMu7KNdaPFAj89D3Waw6QV+kAIpaqABrq6KEt3HuVv87dhMYZh3Zoz8eoQOjWvd6ZBzGJI3gK3fqxhrlQV0UBXF+yL9Qd5KTKGnsH1+e/o7mdu2f9DYR788op15ErXu5xSo1I1kQa6sluxxfDOz3uZ9tt+ru/YlA9Gd8fP2/YszsxkyDpmfb37B8g4DMMjwUO/plGqqmigK7tsOnCClyJj2H00k1G9W/Hq8M54edrCeuci+PYBsBSeeUOHIdD2WucUq1QNpYGuziszr5Dnv91F5PZkmtfzO/tuT4D102D5s9C6H1w1BRDrzUIh1zi1bqVqIg10Va4TOQVM+HQd7VJ+5vOOvvRt3wif3LgzT41NiYUts6HjMBgxA7z9nFqvUjWdBroq0/HMPMbO3MiAk9/wrNcXcADrT2m9J8OQ161PBVJKOZUGujrHzqQMHpq7hcLsdJ6q9T207G99pmdpHl7gV7fK61NKlU0DXf0pLTuffy/fy4LoRBoG+LK06wa8dmXC4H+CfwNnl6eUqoAGugJg/f50Jn8ZzemCYib1C+GRnj7UmTEbLh9jHU+ulKr2NNAVxzLyeHjeFprU8eWTcRG0b1IbFt0L4gkDn3N2eUopO9kV6CIyBHgP6zNFZxpjXi+1/QHgIaAYyAYmG2NiHVyrcrRNMzDr3scrM5cfig2NfH3w/so2tjzzCFzzFNRt7twalVJ2qzDQRcQTmAYMApKAKBGJLBXYc40xH9vaDwPeAYZUQr3KEYyBFVNhzTsk1e7GuoK29GnbAO+GJeZcCWgC/f7mvBqVUhfMniv03kC8MSYBQETmA8OBPwPdGJNZon0AYBxZpLoIlmLIOnruemNg5WuwbQ7xre5gcNxwRvdpw123aj+5Uq7OnkBvASSWWE4CrijdSEQeAh4HfICBDqlOXZys4zD3Dji6vdwmK5tNZELcQK4ObcwLN4dXYXFKqcpiT6BLGevOuQI3xkwDponIaOB5YPw5OxKZDEwGCA4OvrBKlX3S98OXf4GcVOtwQ796Z23OyS/i3c0FzDzQinv7hvDsTWFn5mRRSrk0ewI9CWhVYrklkHye9vOBj8raYIyZDkwHiIiI0G4ZRzuyBebcARgYvwRa9jxrc0JqNvfOiiI5I49/39GF23u2dE6dSqlKYU+gRwGhIhICHAFGAqNLNhCRUGNMnG1xKBCHqlrxK2DBOPBvCOMWQ6PQszZvSEjn/i834+khzLvvCnq21huFlHI3FQa6MaZIRKYAy7EOW/zMGBMjIlOBaGNMJDBFRK4HCoGTlNHdoirRjoXw3YPQOAzGLIK6zc7a/N3WIzy5aDvBDfz5fEJvghvqE4SUckd2jUM3xiwFlpZa92KJ1486uC5lr00zYOkT0OZqGDnnnD7z6av288/7KhwAAA0tSURBVK+le+jTtgGfjI2gnr8+nFkpd6V3irqyEwmw7BkIvQHu/OKs6WstFsO/lu5m5poDDO3ajHfu7Iavl86IqJQ700B3ZSumgqc33PLeOXORv/xDDF+sP8SEq9rw4s3heHiUNVhJKeVOdLyaq0rcBDHfwlWPnNNnHrk9mS/WH2JSvxBeukXDXKmaQgPdFRkDPz0PtZvCVQ+ftelgWg7PLt5Jz9b1+ceNYWceFaeUcnva5eKKdkdC4ka45X3wrf3n6vyiYh6etxVPD+H9Ud3x1huGlKpRNNBdjTGw4lVoEg7dx5ZYbZj6Qyw7j2TwybietAis5cQilVLOoJdwrubwBkiPs3a12J7jaYxh6pJY5mw8zP3XtuWGTkFOLlIp5Qx6he5qts8D7wDoOAywhvkrP8Qya91B7u0bwtNDwpxcoFLKWTTQXUnhaevIlvBhf/ad/9+Pu5m17iCT+oXw3NCO+iWoUjWYdrm4kr1LIT8Tuo0CYOXeFGauOcDdV7bWMFdKaaC7lG3zoG5LaHM1GacLefqbnYQ2qc2zN2mYK6U00F1H1jHYvwK63QUeHry6JJbU7Hz+fUc3/Lz1ln6llAa669j5NRgLdBvFit3HWbQ5iQevbUe3VoHOrkwpVU1ooLsCY6zdLS0iWHasNo8t2EZYUB0evq69sytTSlUjGuiu4NgOSInhe3MND3y1hZBGAcy4O0JnT1RKnUWHLbqAgs1zAC9eOhDGg/3b8figDnpbv1LqHBro1ZwpKiB/6wLWWHrw7vgBDAhr4uySlFLVlF7mVXMrlsylTvEppNsoDXOl1HlpoFdjmw+dpHDLXDI9Arlh+NiK36CUqtHsCnQRGSIie0UkXkSeLmP74yISKyI7RGSFiLR2fKk1y4G0HJ756neu89iMb/e7EC8fZ5eklKrmKgx0EfEEpgE3AuHAKBEJL9VsKxBhjOkKLALedHShNcn2xFPc9tE6BhStwYcifHuOdnZJSikXYM8Vem8g3hiTYIwpAOYDw0s2MMb8ZozJtS1uAFo6tsya4/d9qYyasYEAHw8ea7LZOu95s27OLksp5QLsCfQWQGKJ5STbuvJMBP5X1gYRmSwi0SISnZqaan+VNcS6/WlMnBVFSINaLA/7Eb/jW6DH3aDztCil7GBPoJeVJqbMhiJjgQjgrbK2G2OmG2MijDERjRs3tr/KGuBwei4PzdlCaEMfvg36FP+tn0Kfh6D3/c4uTSnlIuwZh54EtCqx3BJILt1IRK4HngOuNcbkO6a8miE7v4j7vojGw1LI4nof4rNnDQx6Ffo+4uzSlFIuxJ4r9CggVERCRMQHGAlElmwgIt2BT4BhxpgUx5fpviwWw2MLthGfms3CnruplbQGhv1Xw1wpdcEqDHRjTBEwBVgO7AYWGmNiRGSqiAyzNXsLqA18LSLbRCSynN2pUr7ZksTPscd5eVAL2sV+AG37Q/dxzi5LKeWC7Lr13xizFFhaat2LJV5f7+C6aoSc/CLeWr6Xy1sFMrZgEZw+Ze1q0S9BlVIXQe8UdaKPf99PSlY+/+xfB9n0MVw+Gpp1dXZZSikXpZNzVaX8bCguACA5M48Fq3YwsnMTOse+C+IJA55zcoFKKVemgV5VYiNh0b1gKQSgObDJC4i3bb/6Cah3vuH9Sil1fhroVaEwD5Y/Cw3bQ88JHDqRw+drD3Jth8YMuKwJ+NaBLnc4u0qllIvTQK8KGz+GjES4O5K8Vv2Y8N5qCut256lR14CPngKllGPol6KVLScdVr8NoTdA22v5zy9xHEjL4fURXfHXMFdKOZAGemX7/Q0oyIZBU9mZlMGM1QncFdGKfqGNnF2ZUsrN6CWio6Xugx0LwBSDpRiiP4Ue4ylo0IEnP1hDo9o+PDu0o7OrVEq5IQ10Rzq0DuaOhIIs8LD9p60fgun/NC9FxrDnWBYz7o6gXi1v59aplHJLGuiOsnuJdVhiYDCMW2P9bfPBijjmbTrMX/u3Y1B4UycWqZRyZxro9jLG+uXm/l/L2GaBxI3QoieMXgj+Df7c9HV0Im//vI8R3Vvw5A2XVWHBSqmaRr8UtUdxIXz3IPz6KhTmgnic/ePhBT3Gw93f/xnmRcUWZq09wDOLd9KvfSNev60ronO0KKUqkV6hVyQ/G74eD/G/wIDn4ZonKpw8a/3+dF75wdpn3q99Iz4a2wMfL/1/p1KqcrleoK99H355ueo+z1isAX7L+9Bz/PmbGsPr/9vDJ6sSaBFYi4/G9GBI5yC9MldKVQnXC/SWEdDvsar9zLb9IeTq8zYpthie/24n8zYlMuaKYJ4fGk4tH88qKU8ppcAVA731VdafaqSw2MLfF24ncnsyDw1oxxODL9OrcqVUlXO9QK9msvIKeXjeVlbuTeUfQ8J4sH87Z5eklKqhNNAvQfKp09w7K4q4lGxeG9GFUb2DK36TUkpVEg30i7TpwAmmzN3C6YJiZt3Ti6tDGzu7JKVUDWfXWDoRGSIie0UkXkSeLmP7NSKyRUSKROR2x5dZfUQdPMHYmRu585P1eHt68M1fr9IwV0pVCxVeoYuIJzANGAQkAVEiEmmMiS3R7DAwAXiiMop0przCYrYcOsnq+DRW7UslJjnTOsHWTWGM7dNap8BVSlUb9qRRbyDeGJMAICLzgeHAn4FujDlo22aphBqrhDGGY5l57D6aye6jWew5lsWeo5kkpOVQbDF4eQjdgwN58eZwRvUO1iGJSqlqx55AbwEkllhOAq64mA8TkcnAZIDg4Iv7AnHm6gTeXLb3ot77B19vD/x9PPH38aKw2EJuQTE5+UXkF535/1GLwFp0bFaXIZ2D6NYykD7tGlLbV6/GlVLVlz0JVdaAanMxH2aMmQ5MB4iIiLiofXRpUY+JV4dczFttNVi7UU4XFJNbWIy3h+Dvaw33VvVrEdasLpcF1aGun05xq5RyLfYEehLQqsRySyC5csqp2BVtG3JF24bO+nillKq27BnlEgWEikiIiPgAI4HIyi1LKaXUhaow0I0xRcAUYDmwG1hojIkRkakiMgxARHqJSBJwB/CJiMRUZtFKKaXOZde3fMaYpcDSUuteLPE6CmtXjFJKKSfRSbqVUspNaKArpZSb0EBXSik3oYGulFJuQgNdKaXchBhzUTdsXvoHi6QChy7y7Y2ANAeW4ypq4nHXxGOGmnncNfGY4cKPu7UxpswpXp0W6JdCRKKNMRHOrqOq1cTjronHDDXzuGviMYNjj1u7XJRSyk1ooCullJtw1UCf7uwCnKQmHndNPGaomcddE48ZHHjcLtmHrpRS6lyueoWulFKqFA10pZRyEy4X6CIyRET2iki8iDzt7Hoqg4i0EpHfRGS3iMSIyKO29Q1E5GcRibP9ru/sWh1NRDxFZKuILLEth4jIRtsxL7DNye9WRCRQRBaJyB7bOb+yhpzrx2x/vneJyDwR8XO38y0in4lIiojsKrGuzHMrVu/bsm2HiPS40M9zqUAXEU9gGnAjEA6MEpFw51ZVKYqAvxtjOgJ9gIdsx/k0sMIYEwqssC27m0exzrv/hzeAd23HfBKY6JSqKtd7wDJjTBjQDevxu/W5FpEWwCNAhDGmM+CJ9eE57na+ZwFDSq0r79zeCITafiYDH13oh7lUoAO9gXhjTIIxpgCYDwx3ck0OZ4w5aozZYnudhfUveAusxzrb1mw2cKtzKqwcItISGArMtC0LMBBYZGvijsdcF7gG+BTAGFNgjDmFm59rGy+gloh4Af7AUdzsfBtjVgEnSq0u79wOB74wVhuAQBFpdiGf52qB3gJILLGcZFvntkSkDdAd2Ag0NcYcBWvoA02cV1ml+A/wFGCxLTcETtmemgXueb7bAqnA57auppkiEoCbn2tjzBHg38BhrEGeAWzG/c83lH9uLznfXC3QpYx1bjvuUkRqA98AfzPGZDq7nsokIjcDKcaYzSVXl9HU3c63F9AD+MgY0x3Iwc26V8pi6zceDoQAzYEArF0Opbnb+T6fS/7z7mqBngS0KrHcEkh2Ui2VSkS8sYb5HGPMYtvq43/8E8z2O8VZ9VWCvsAwETmItSttINYr9kDbP8nBPc93EpBkjNloW16ENeDd+VwDXA8cMMakGmMKgcXAVbj/+Ybyz+0l55urBXoUEGr7JtwH65cokU6uyeFsfcefAruNMe+U2BQJjLe9Hg98X9W1VRZjzDPGmJbGmDZYz+uvxpgxwG/A7bZmbnXMAMaYY0CiiFxmW3UdEIsbn2ubw0AfEfG3/Xn/47jd+nzblHduI4G7baNd+gAZf3TN2M0Y41I/wE3APmA/8Jyz66mkY+yH9Z9aO4Bttp+bsPYprwDibL8bOLvWSjr+/sAS2+u2wCYgHvga8HV2fZVwvJcD0bbz/R1Qvyaca+AVYA+wC/gS8HW38w3Mw/odQSHWK/CJ5Z1brF0u02zZthPrCKAL+jy99V8ppdyEq3W5KKWUKocGulJKuQkNdKWUchMa6Eop5SY00JVSyk1ooCullJvQQFdKKTfx/4G3OK242elfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores2)\n",
    "plt.plot(test_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tanh + Sigmoid + Cross Entropy Cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net3(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net3, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(28*28, 30)\n",
    "        self.fc2 = nn.Linear(30, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.tanh(self.fc1(x)) # weight参数不知道怎么用，随便写的一个\n",
    "        x = F.logsigmoid(self.fc2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\yinahe\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\torch\\nn\\functional.py:1340: UserWarning: nn.functional.tanh is deprecated. Use torch.tanh instead.\n",
      "  warnings.warn(\"nn.functional.tanh is deprecated. Use torch.tanh instead.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net3 = Net3()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net3.parameters(), lr = 1e-1)\n",
    "loss_scores3, train_scores3, test_scores3 = fit(net3, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3hUZfr/8fedQu9dCBAVpEkflZIQFxABURRRsOHaEEW666r73XXXLa6u0gQpgl2xAKKAgCxgCEUk9KpUEaVEQAigQOD5/ZHhJ7IJJGSGk5n5vK4rF5mchzn32cN+PLnnOc8x5xwiIhL6orwuQEREAkOBLiISJhToIiJhQoEuIhImFOgiImEixqsdlytXzsXHx3u1exGRkLRs2bIfnXPls9rmWaDHx8eTmprq1e5FREKSmX2b3Ta1XEREwoQCXUQkTCjQRUTChAJdRCRMKNBFRMKEAl1EJEwo0EVEwsR5A93MqprZPDPbYGbrzKxfFmM6m9lqM1tpZqlmlhCccmH/keM8O3U9R49nBGsXIiIhKSdX6BnAIOdcHaAZ0NvM6p41Zg7Q0DnXCLgfGBfYMn+1cPOPvL5oGzePXMiWtMPB2o2ISMg5b6A753Y555b7v08HNgBVzhpz2P36pIyiQNCemnFjw8q8ff81/Hj4ODe9vIBpq38I1q5EREJKrnroZhYPNAaWZLHtFjPbCEwn8yo9q7/f09+SSU1LS8t9tX4JNcsxvW8CtSoV57H3VvDMJ2s5lnHygt9PRCQcWE4fQWdmxYBk4J/OucnnGNcK+Itzru253s/n87m8ruVyPOMUz8/cyPgF26hfpSQj7mxM9bJF8/SeIiL5mZktc875stqWoyt0M4sFJgHvnivMAZxz84HLzaxcrivNpQIxUfy5U11e7eFjx/6jdBquFoyIRK6czHIxYDywwTk3OJsxNfzjMLMmQAFgXyALPZfr6lZket8EalYsxmPvreDJSav5+bhaMCISWXKyfG5L4B5gjZmt9P/saaAagHNuNHAr0MPMTgA/A91cTns5ARJXuggfPNycIbO/YVTyFlK/PcDw7o2pW7nExSxDRMQzOe6hB1ogeujZWbj5R/p/sJKDR0/wZIfa3NcyHv8vECIiIS3PPfRQ07JGOWb2SySxZjmenbae37++lLT0Y16XJSISVGEZ6ABlixVk3L0+nu1cjy+37qP90Pn8d/0er8sSEQmasA10ADOjR/N4pvZJoEKJQjz4VipPTV6jZQNEJCyFdaCfdkXF4kzp3YKHky7j/aU76DgsheU7DnhdlohIQEVEoAMUjInmqQ51mPBQM06cdHQdtYgXZ33N8YxTXpcmIhIQERPopzW7rCwz+yfSpUkcI+Zt5uaRC9m4+5DXZYmI5FnEBTpA8UKxvHhbQ8bc05S96b9w08sLGfXFFk6e8mYKp4hIIERkoJ92fb1KzOrfijZ1KvD8zI10Hb1IS/KKSMiK6ECHzOmNr9zVhGHdG7E17Qgdh6UwLmWrrtZFJOREfKBD5vTGzo2qMHtAKxJrluMf0zfQbcxitv14xOvSRERyTIF+hgolCvFqDx+Db2/IN3vSaT90vq7WRSRkKNDPYmZ0aRLH7IFJJNTIvFq/fcxiNu9Vb11E8jcFejYqlijEuHt9DOnWkM17D9NxeAqjvthCxknNWxeR/EmBfg5mxi2N45g9sBW/q1We52du5JZXFrFhl+ati0j+o0DPgQrFCzH67qaMvLMJuw7+zI0vL2Dw51/rOaYikq8o0HPIzLihwSXMHpDEjQ0rM3zuZjoNX8Cyb7UmjIjkDwr0XCpdtABDujXi9d9fxZFjGXQdvYi/TV3HkWNawVFEvKVAv0C/q12BzwcmcU+z6ry+cDvthszni6/3el2WiEQwBXoeFCsYw7Odr+SjXs0pFBvF719fyoAPVrL/yHGvSxORCKRAD4Cr4svwWb9E+rapybTVP9DmpS+YvHwnXj2vVUQikwI9QArGRDPwuiuY1ieR+HJFGfjhKnq89hXf7T/qdWkiEiEU6AFWq1JxJvZqwd9uqsfybw/Qbsh8xs7XDUkiEnznDXQzq2pm88xsg5mtM7N+WYy5y8xW+78WmVnD4JQbGqKjjHtbxDN7YBIta5TjX59tpPPIhaze+ZPXpYlIGMvJFXoGMMg5VwdoBvQ2s7pnjdkGJDnnGgB/B8YGtszQVLlUYV7t0ZTRdzchLf0YN49cyN+mruOwpjiKSBCcN9Cdc7ucc8v936cDG4AqZ41Z5Jw7fYfNl0BcoAsNVWZG+ysv4b+Dkrjzmmq8sWg77QYnM3v9Hq9LE5Ewk6seupnFA42BJecY9gAwI5u/39PMUs0sNS0tLTe7DnklCsXyj5vrM7FXC4oXiuWht1J5+O1Udh/8xevSRCRMWE6n1plZMSAZ+KdzbnI2Y34HvAIkOOf2nev9fD6fS01NzWW54eHEyVOMnb+V4XM2ERsdxePtruCe5vFER5nXpYlIPmdmy5xzvqy25egK3cxigUnAu+cI8wbAOKDz+cI80sVGR9H7dzWYPSCJxtVK8dep67nllYWs/f6g16WJSAjLySwXA8YDG5xzg7MZUw2YDNzjnPsmsCWGr2pli/DW/VczrHsjfvjpF24asYBnp67Xh6YickHO23IxswQgBVgDnJ5M/TRQDcA5N9rMxgG3At/6t2dk9yvBaZHccsnKwZ9P8MLMjby7ZAeXlCzEMzfW4/p6Fcn876mISKZztVxy3EMPNAV61pZ9e4A/fbyGjbvTaVunAn+9qR5xpYt4XZaI5BN57qHLxdO0emmm9kng6Y61Wbh5H9cNns+Y5C2c0J2mInIeCvR8KDY6ip6tLmf2wFa0rFGW52ZspNPwBaRu3+91aSKSjynQ87G40kUYd+9VjL2nKem/nKDr6MU8OWk1B7Q8r4hkQYEeAtrVq8TsgUn0bHUZHy3bSZvByXyY+p2W5xWR31Cgh4iiBWN4umMdpvdN4LJyRXli4mpuH7OYr3ene12aiOQTCvQQU7tSCT58uDkv3NqAzXsPc8PwFJ6bsYGjxzV3XSTSKdBDUFSUcftVVZkz6Fq6NKnCmOSttH0pmVnrdqsNIxLBFOghrEzRArzQtSETezWnROFYHn57GQ+8maqnJIlEKAV6GPDFl2FqnwT+1LEOX27dR9vByYyYu4ljGSe9Lk1ELiIFepiIjY7ioVaXMWdQEm3qVODFz7+hw9AUFmz60evSROQiUaCHmUtKFuaVu5ryxn1XcdI57h6/hD4TVrDnkNZdFwl3CvQwdW2tCszq34r+bWsya91u2ryUzPgF2/SwapEwpkAPY4Vio+nf9go+79+KptVL8/dp6+n0spYQEAlXCvQIEF+uKG/cdxWj727CoZ8zlxD4w0er2Hf4mNeliUgAKdAjxOmHVc8emESvpMv5eMX3tH4pmXe+/JaTpzR3XSQcKNAjTNGCMTzZoTYz+iVS55Li/N+UtdzyykJW7/zJ69JEJI8U6BGqZsXiTHioGcO6N2LXwV/oPHIhf/p4DT8d1UqOIqFKgR7BzIzOjaowd1AS97W4lPeXfkfrlzJXcjylNoxIyFGgC8ULxfKXG+sy9bFfV3K8bcxi1v1w0OvSRCQXFOjy/9WtnLmS44u3NWT7j0e48eUF/PXTdRz8+YTXpYlIDijQ5TeiooyuTeOY+/i13N2sOm8t3k6bl75g0rKdWslRJJ9ToEuWShaO5dnOV/LpYwnElS7CoI9WcfuYxWzYdcjr0kQkG+cNdDOrambzzGyDma0zs35ZjKltZovN7JiZPR6cUsULV1YpyeRHWvD8rfXZvPcwnV5ewLNT13PoF7VhRPKbnFyhZwCDnHN1gGZAbzOre9aY/UBf4MUA1yf5QFSU0e2qasx7/Fq6XVWV1xdto81LyUxZ8b3aMCL5yHkD3Tm3yzm33P99OrABqHLWmL3OuaWALtvCWKkiBfjXLfX5pHdLKpcsRP8PVtJt7Jds3K02jEh+kKseupnFA42BJReyMzPraWapZpaalpZ2IW8h+UCDuFJ8/GhLnutSn0170rlh+AL+Pm096WrDiHgqx4FuZsWASUB/59wFXZI558Y653zOOV/58uUv5C0kn4iKMu64uhpzB2W2YV5buI3WasOIeCpHgW5msWSG+bvOucnBLUlCSemimW2YKY/+tg3z9e50r0sTiTg5meViwHhgg3NucPBLklDUsOqvbZhv9qTTcXiK2jAiF5md79djM0sAUoA1wOnH3TwNVANwzo02s0pAKlDCP+YwUPdcrRmfz+dSU1PzfACS/xw4cpz/fP41E77aQbliBflTxzp0blSZzGsDEckLM1vmnPNluc2rfqcCPfyt3vkTf56yllU7D3L1pWX4e+crqVWpuNdliYS0cwW67hSVoDk9G+bf/tkwasOIBJcCXYIqKsrortkwIheFAl0uitOzYXRTkkjwKNDlosrqpiStDSMSGAp0uejOvCmpu39tmNYvJjN5uZboFckLBbp4pnTRAvzT34apUrowAz9cRbcxX2qJXpELpEAXzzWIK8XHj7TInA2zN51OLy/gb1PXqQ0jkksKdMkXTs+Gmfd4ZhvmjUXbaf1iMh+vUBtGJKcU6JKvlCry2zbMgA/UhhHJKQW65Etqw4jkngJd8q3s2jCaDSOSNQW65Htnt2FOz4bRTUkiv6VAl5BxdhtGT0oS+S0FuoQUrQ0jkj0FuoSk7NaG0ZOSJJIp0CWknbk2jJ6UJJFOgS4h7/TaMPMGXcvtvjheW7iNNi8l8+mqH9SGkYiiQJewUbpoAZ7r0oCPH21JxRKF6DthBXeNW8LmvWrDSGRQoEvYaVS1FFN6t+QfN1/Juh8O0X5oCs/N2MCRYxlelyYSVAp0CUvRUcbdzaozd1AStzSuwpjkrbQdnMyMNbvUhpGwpUCXsFa2WEH+c1tDJj3SnFJFCvDIu8vp8dpXbE077HVpIgGnQJeI0LR6GaY+1pK/3liXlTt+ov3QFF6c9TU/Hz/pdWkiAXPeQDezqmY2z8w2mNk6M+uXxRgzs+FmttnMVptZk+CUK3LhYqKj+H3LS5nzeBI3NLiEEfM2c92QZGav3+N1aSIBkZMr9AxgkHOuDtAM6G1mdc8a0wGo6f/qCYwKaJUiAVSheCGGdGvEhIeaUTg2mofeSuXBN5fy3f6jXpcmkifnDXTn3C7n3HL/9+nABqDKWcM6A2+5TF8CpczskoBXKxJAzS8vy/S+iTzVoTaLtuyj7eBkRszdxLEMtWEkNOWqh25m8UBjYMlZm6oA353xeif/G/qYWU8zSzWz1LS0tNxVKhIEBWKieDjpcv47MInWtSvw4uff0GFoCgs2/eh1aSK5luNAN7NiwCSgv3Pu7HVLLYu/8j9zw5xzY51zPuecr3z58rmrVCSIKpcqzKi7m/Lm/VdzyjnuHr+Ex95bzp5Dv3hdmkiO5SjQzSyWzDB/1zk3OYshO4GqZ7yOA37Ie3kiF1fSFeWZ2b8VA9pewefr99DmpWReW7CNjJOnvC5N5LxyMsvFgPHABufc4GyGfQr08M92aQYcdM7tCmCdIhdNodho+rWtyewBrWhavTTPTlvPjSMWsuzbA16XJnJOdr675swsAUgB1gCnL1OeBqoBOOdG+0N/BNAeOArc55xLPdf7+nw+l5p6ziEinnPOMXPtbp6dtp5dB3+h+1VV+WP72pQuWsDr0iRCmdky55wvy21e3QatQJdQcuRYBsPmbGL8gm2UKBTDUx3q0LVpHFFRWX18JBI85wp03SkqkgNFC8bwdMc6TO+bQI0KxXhi0mpuH7NYzzWVfEWBLpILtSuV4IOezXmhawO2pB3mhuEL+NdnWslR8gcFukguRUUZt/uqMtf/QI2x87dy3eBkZq3brZUcxVMKdJELdPqBGhN7NadE4VgefnsZD72Vys4DWkJAvKFAF8kjX3wZpvZJ4OmOmUsIXDd4PqO+2MIJzV2Xi0yBLhIAsdFR9Gx1ObMHJpFYsxzPz9zIDcNTWLp9v9elSQRRoIsEUJVShRnbw8erPXwcOXaS20Yv5o8TV3PgyHGvS5MIoEAXCYLr6lZk9sBWPJx0GZOW76TN4GQmLtupD00lqBToIkFSpEDmDUjT+iYQX7YIj3+0iu5jv2TzXj3+ToJDgS4SZLUrlWBirxY816U+G3YdosOw+Qz+/Gt+OaF11yWwFOgiF0FUlHHH1dWYM+habqh/CcPnbqbDsBQWbta66xI4CnSRi6h88YIM7d6Ydx64Buccd41bwsAPVrLv8DGvS5MwoEAX8UBCzXLM7N+Kvq1rMHX1D7R+KZkPlu7Qh6aSJwp0EY8Uio1mYLtazOiXSK2KxfnjpDV0G/slm/eme12ahCgFuojHalQozvs9m/H8rfX5enc6HYalMHj2N/rQVHJNgS6SD0RFGd2uqsacQUmZH5rO2UTHYSks3rLP69IkhCjQRfKRcsUyPzR96/6ryTjluOPVL3li4ip+Oqo7TeX8FOgi+VCrK8ozq38reiVdzqTl39PmpWQ+Wfm9PjSVc1Kgi+RThQtE82SH2kx9LIG4MkXo9/5K7n19Kd/t1/K8kjUFukg+V7dyCSY/0oJnbqzLsu37aTdkPmPnbyFDy/PKWRToIiEgOsq4r+WlzB6YRMsaZfnXZxvpPHIha78/6HVpko8o0EVCSOVShXm1h49X7mrC3vRj3DQi85mmPx/XFEfJQaCb2WtmttfM1mazvbSZfWxmq83sKzO7MvBlishpZkbH+pfw34FJdLuqKmPnb6Xd0GRSNqV5XZp4LCdX6G8A7c+x/WlgpXOuAdADGBaAukTkPEoWjuW5Lg14v2czYqKiuGf8Vwz8cKUephHBzhvozrn5wLmeo1UXmOMfuxGIN7OKgSlPRM6n2WVlmdEvkd6/u5xPV/5A28Ga4hipAtFDXwV0ATCzq4HqQFxWA82sp5mlmllqWpp+PRQJlEKx0fzh+tpM7ZNAXOnC9Ht/JQ+8mcr3P/3sdWlyEQUi0P8NlDazlUAfYAWQkdVA59xY55zPOecrX758AHYtImeqc0kJJj/akv+7oQ6Lt+yj3eBk3lq8nVOndLUeCfIc6M65Q865+5xzjcjsoZcHtuW5MhG5INFRxoOJl/H5gFY0qV6av3yyjtvHLNaj7yJAngPdzEqZWQH/yweB+c65Q3l9XxHJm6plivDW/Vfz0m0N2Zx2mI7DUhgxdxMndENS2MrJtMUJwGKglpntNLMHzKyXmfXyD6kDrDOzjUAHoF/wyhWR3DAzbm0ax+wBSVxXryIvfv4NN41YyJqduiEpHJlXn4T7fD6Xmprqyb5FItWsdbv585S17DtynAcTL2VA2ysoFBvtdVmSC2a2zDnny2qb7hQViSDX16vE7IFJdG0Sx5jkrXQYlsKSrVpzPVwo0EUiTMnCsTzftQHvPngNGadO0W3sl/x5yloOH8tycpqEEAW6SIRqWaMcs/q34r6W8byz5FuuHzKf5G90f0goU6CLRLAiBWJ45sZ6TOzVnEKxUdz72lf84aNVHDx6wuvS5AIo0EWEptXLML1v5vIBk1d8z3VDkpm9fo/XZUkuKdBFBPh1+YBPerekTNECPPRWKn0mrGC/FvsKGQp0EfmNK6uU5NPHEhjQ9gpmrt3FdYOTmb56l9dlSQ4o0EXkfxSIiaJf25pM7ZNA5VKF6f3ech55Zxlp6ce8Lk3OQYEuItmqXakEHz/agifa12LOhr20G6KlefMzBbqInFNMdBSPXluD6X0TqF62KP3eX8nDby9jb/ovXpcmZ1Ggi0iO1KxYnEmPtOCpDrX54ps02g2Zr6v1fEaBLiI5Fh1lPJx0OZ/1TeTScplX6z11tZ5vKNBFJNdqVCjGxF4teLpjbZJ1tZ5vKNBF5IJERxk9W/32av3htzUTxksKdBHJk9NX67/21pOZuuoHr8uKSAp0EcmzX3vrCVQrU4Q+E1bQ+93l7Dusq/WLSYEuIgFTo0LmTJg/XF+Lz9fvpt2Q+cxcq7tMLxYFuogEVEx0FL1/V4OpfRKoVLIQvd5ZTr/3V/DTUa0JE2wKdBEJitqVSjCld0v6t63J9NW7aDdkPvM27vW6rLCmQBeRoImNjqJ/2yuY0rslpYsU4L43lvLExFWk/6L11oNBgS4iQXdllZJ82qclj1x7OROX7aT90BQWbf7R67LCjgJdRC6KgjHR/LF9bSY+0oKCMVHcOW4Jz3yylp+Pn/S6tLBx3kA3s9fMbK+Zrc1me0kzm2pmq8xsnZndF/gyRSRcNKlWmul9E/l9i3jeXPwtHYensHzHAa/LCgs5uUJ/A2h/ju29gfXOuYbAtcBLZlYg76WJSLgqXCCav95Uj/ceuobjGafoOmoRL8zcyPGMU16XFtLOG+jOufnA/nMNAYqbmQHF/GMzAlOeiISzFpeXY2b/RLo2jeOVL7bQeeRCNu4+5HVZISsQPfQRQB3gB2AN0M85l+V/Zs2sp5mlmllqWlpaAHYtIqGueKFYXujakFd7+EhL/4WbXl7I6OQtnDylhb5yKxCBfj2wEqgMNAJGmFmJrAY658Y653zOOV/58uUDsGsRCRfX1a3IrP6taF27Av+esZHuYxezY99Rr8sKKYEI9PuAyS7TZmAbUDsA7ysiEaZssYKMursJg29vyMZd6bQfNp8JX+3Qsrw5FIhA3wG0ATCzikAtYGsA3ldEIpCZ0aVJHDMHtKJR1VI8NXkND76Zqodo5EBOpi1OABYDtcxsp5k9YGa9zKyXf8jfgRZmtgaYA/zROac7BkQkT6qUKsw7D1zDXzrVZcHmH2k/NIWZa3d7XVa+Zl79KuPz+Vxqaqon+xaR0LJpTzoDPlzJ2u8P0bVpHM/cWJfihWK9LssTZrbMOefLapvuFBWRfK9mxeJMfqQlfVrXYPLynXQYlsJX2841mzoyKdBFJCQUiIliULtafNSrBdFRRrexi/n3jI0cy9DSAacp0EUkpDStXprP+ibS/aqqjE7ews0jF/HNnnSvy8oXFOgiEnKKFozhuS4NeLWHj72HfqHTywsYv2AbpyL8ZiQFuoiErOvqVmRm/1Yk1ijH36etp8drX7H7YOROb1Sgi0hIK1+8IOPu9fGvW+qz7NsDXD90PtNXR+ZzTBXoIhLyzIw7r6nG9L4JxJcrSu/3ljPow8h7MpICXUTCxmXlizGxV3P6tq7Bxyt20nF4CqnbI2d6owJdRMJKbHQUA9vV4qNezTGM28csZvDnX3PiZPivta5AF5Gw1LR6GT7rl0iXJnEMn7uZrqMXs+3HI16XFVQKdBEJW8UKxvDibQ0ZeWcTtv94hBuGp/Dh0u/CdvVGBbqIhL0bGlzCzP6JNIwrxROTVvPou8v56ehxr8sKOAW6iESES0oW5p0Hr+HJDrX574Y9tB+awqLN4bUwrAJdRCJGdJTRK+lyPn60JUUKRnPX+CU899mGsHk4tQJdRCLOlVVKMq1PAndcXY0x87fSZdRCtqQd9rqsPFOgi0hEKlIghn/dUp8x9zRl54Gf6TR8Ae+H+OPuFOgiEtGur1eJWf1b0aR6KZ6cvIZH3gndD0wV6CIS8SqWKMTb91/D0x1rM2dj5gemi7fs87qsXFOgi4gAUVFGz1aXM/mRlhQuEM2d477kP7M2htQdpgp0EZEz1I/L/MD0tqZxjJy3hdtGL2bHvqNel5UjCnQRkbMULRjDC10bMuLOxmxJO0zH4SlMWfG912WdlwJdRCQbnRpUZka/RGpXKk7/D1Yy8IOVHD6W4XVZ2TpvoJvZa2a218zWZrP9D2a20v+11sxOmlmZwJcqInLxxZUuwvs9m9G/bU2mrPyeG4ansOq7n7wuK0s5uUJ/A2if3Ubn3H+cc42cc42Ap4Bk51zkLEAsImEvJjqK/m2v4IOHm5Nx0nHrqEWMTt6S755het5Ad87NB3Ia0HcAE/JUkYhIPnVVfBk+65tIu3oV+feMjdz7+lfsTc8/zzANWA/dzIqQeSU/6RxjeppZqpmlpqWlBWrXIiIXTckisYy8swnPdanP0u376TA0hXlf7/W6LCCwH4reCCw8V7vFOTfWOedzzvnKly8fwF2LiFw8ZsYdV1dj6mMJlC9ekPteX8o/pq3nWMZJT+sKZKB3R+0WEYkgNSsWZ0rvlvRoXp1xC7Zx66hFnj4VKSCBbmYlgSTgk0C8n4hIqCgUG82zna9kzD1N+W7/z3QansLk5Ts9qSUn0xYnAIuBWma208weMLNeZtbrjGG3AJ8758L7gX0iItm4vl4lZvRLpF7lkgz8cJUnc9bNq6UifT6fS01N9WTfIiLBknHyFC/P3czLczdRvWxRXr6jMVdWKRmw9zezZc45X1bbdKeoiEgAxURHMeC6K3jvoWb8fPwkXV5ZxOsLt12UddYV6CIiQdDssrJ81i+RxJrl+NvU9Tz0VioHjgR3nXUFuohIkJQpWoBx9/r4S6e6JH+TRsfhKXy1LXg30ivQRUSCyMy4P+FSJj/SkoIxUXQfu5jxC7YFZV8KdBGRi6B+XEmm9U2kc6MqXFauaFD2EROUdxURkf9RrGAMQ7o1Ctr76wpdRCRMKNBFRMKEAl1EJEwo0EVEwoQCXUQkTCjQRUTChAJdRCRMKNBFRMKEZ8vnmlka8O0F/vVywI8BLCdUROJxR+IxQ2QedyQeM+T+uKs757J8hqdngZ4XZpaa3XrA4SwSjzsSjxki87gj8ZghsMetlouISJhQoIuIhIlQDfSxXhfgkUg87kg8ZojM447EY4YAHndI9tBFROR/heoVuoiInEWBLiISJkIu0M2svZl9bWabzexJr+sJBjOrambzzGyDma0zs37+n5cxs9lmtsn/Z2mvaw0GM4s2sxVmNs3/+lIzW+I/7g/MrIDXNQaSmZUys4lmttF/zptHwrk2swH+f99rzWyCmRUKx3NtZq+Z2V4zW3vGz7I8v5ZpuD/fVptZk9zsK6QC3cyigZFAB6AucIeZ1fW2qqDIAAY55+oAzYDe/uN8EpjjnKsJzPG/Dkf9gA1nvH4eGOI/7gPAA55UFTzDgJnOudpAQzKPPazPtZlVAfoCPufclUA00J3wPNdvAO3P+ll257cDUNP/1RMYlZsdhVSgA1cDm51zW51zx4H3gc4e1xRwzrldzrnl/jUSHnIAAAKCSURBVO/Tyfw/eBUyj/VN/7A3gZu9qTB4zCwOuAEY539tQGtgon9IWB23mZUAWgHjAZxzx51zPxEB55rMR2AWNrMYoAiwizA81865+cD+s36c3fntDLzlMn0JlDKzS3K6r1AL9CrAd2e83un/Wdgys3igMbAEqOic2wWZoQ9U8K6yoBkKPAGc8r8uC/zknMvwvw63c34ZkAa87m8zjTOzooT5uXbOfQ+8COwgM8gPAssI73N9puzOb54yLtQC3bL4WdjOuzSzYsAkoL9z7pDX9QSbmXUC9jrnlp354yyGhtM5jwGaAKOcc42BI4RZeyUr/p5xZ+BSoDJQlMx2w9nC6VznRJ7+vYdaoO8Eqp7xOg74waNagsrMYskM83edc5P9P95z+tcv/597vaovSFoCN5nZdjLbaa3JvGIv5f+1HMLvnO8EdjrnlvhfTyQz4MP9XLcFtjnn0pxzJ4DJQAvC+1yfKbvzm6eMC7VAXwrU9H8SXoDMD1E+9bimgPP3jccDG5xzg8/Y9Clwr//7e4FPLnZtweSce8o5F+eciyfz3M51zt0FzAO6+oeF1XE753YD35lZLf+P2gDrCfNzTWarpZmZFfH/ez993GF7rs+S3fn9FOjhn+3SDDh4ujWTI865kPoCOgLfAFuAP3ldT5COMYHMX7NWAyv9Xx3J7CfPATb5/yzjda1B/N/gWmCa//vLgK+AzcBHQEGv6wvwsTYCUv3newpQOhLONfA3YCOwFngbKBiO5xqYQObnBCfIvAJ/ILvzS2bLZaQ/39aQOQsox/vSrf8iImEi1FouIiKSDQW6iEiYUKCLiIQJBbqISJhQoIuIhAkFuohImFCgi4iEif8Hg0exWOxmji4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores3)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxV9Z3/8dcnO9lIQhayEBIgIMhOQNyrouIGdrqBWrW18utMra3TX2fa6a+djm1nap1p60xpp9a6/lSsSy1SLO24axUT9h1C9oUs3Jt9vbnf+eNcSAgJucC9ubnnfp6PB4/kLPfcz3kcfHv4nu/3e8QYg1JKqeAXFugClFJK+YYGulJK2YQGulJK2YQGulJK2YQGulJK2UREoL44NTXV5OXlBerrlVIqKG3btq3JGJM23LaABXpeXh7FxcWB+nqllApKIlIx0jZtclFKKZvQQFdKKZvQQFdKKZvQQFdKKZvQQFdKKZvQQFdKKZvQQFdKKZvQQFdKqTFytLGdn/3lMIeOtfnl+AEbWKSUUnbn6nezv66V90ua2LSrjv11rYhAakI0syYn+Pz7NNCVUsqHWrr62LLvGFv2HuPjMgdtPS4AFk5J4rs3z+Hm+ZlkJMb45bs10JVS6jwYYzjS0M7W0uO8c7iJdw830tvvZkrKBG5ZmMVFeclcHnWYlDAH4ICavdA/D5LzfF6LBrpSSp0FYwyH6tv46OhxtpY52FrmwNHRC0DWxBg+f/FUVi3IYn7ORARgy3fgo/WnHuSmn8LSe3xemwa6Ukp5aX9tK9/fuI+Pyx0AZCdN4BOz0lieP4nl0yYxJWUCImLtbAz86duw9VewbB0svnPgQAlZfqlPA10ppUbR0NbN+jdLeOajCiZOiOT7t8zhmtkZTJkYCZUfQn8TOLH+nHBgE2x7ApZ/Ba7/EZwIej/yKtBFZCXwCBAOPGaM+fGQ7T8DrvIsxgLpxpgkXxaqlFJjydnRy5/3H+O1XXX89WgTAHcsn8rfXzuTpNgo6OuG59dCyV9GPsjF98F1PxyTMAcvAl1EwoH1wLVANVAkIhuNMftP7GOMeWDQ/l8FFvmhVqWU8pvuvn4OHmvjw6PHefNgPdsqnLgNTJ0Uy1eumsGti7KZnhZv7dzXBRtuh6NvwHU/ginLTj9gVBykzxmzMAfv7tCXASXGmFIAEdkArAb2j7D/WuCffVOeUkr5T7Wzk6c/rOC9I00cqW/D5TYAzM1O5L6rC7h2dgZzsxORvi6o2wUVxmobf/dhKH0bVv0CFn8+sCcxiDeBng1UDVquBi4abkcRmQrkA2+ef2lKKeUbxhi2VThpaOsBwOU2bNl3jNf31CEiXDJ9ElfNmsa87Iksnpp8ej/xzf8Xdj47aIXA6vWw6PaxOwkveBPow/17wYyw7xrgJWNM/7AHElkHrAPIzc31qkCllDpXDW3dvLythheKKik/3nnKtoSYCO69Yhp3XZxHVtKEkQ/SVAK7noeFt8P8z3o+nAlps/xY+bnxJtCrgSmDlnOA2hH2XQN8ZaQDGWMeBR4FKCwsHOl/CkopdU7auvt4/0gTH5YeZ2upg0P11pwpy/JSuP+aAi7Mmnhy3ykpE4iN8iIC3/0JRMTAiu9DfLp/CvcRbwK9CCgQkXygBiu0bxu6k4jMApKBD31aoVJKncGxlm4276njjYP1bC114HIbYqPCWTI1mVULs7j+wsnMSI8/t4M3HoY9L1q9VcZ5mIMXgW6McYnIfcAWrG6Ljxtj9onIg0CxMWajZ9e1wAZjjN55K6V8yhhDSUM7e2tbcPVbEdPa7WLLvmMUlTswBmZmxPOly6dx9QXpLMpNIjLcB5PJvvMQREyAS792/scaA171QzfGbAY2D1n3vSHL3/ddWUqpUNXvNhyoa+VoYzvlTZ0cqm9la6mD457h9YPNSI/ngRUzuWVBFvmpcacfrNMBHY3nVkhrLex9GS77OsSlntsxxpiOFFVKjQsdPS5eLK7i8Q/KqXRYDzBFICd5AlfOTOOiaSkszk0mJjIcgMjwMDISoweG2g9V+g48vwb6Ooff7o2oeLjk/nP//BjTQFdKBVRdSxdP/rWc57dW0trtYnFuEl9fYT3AnDop9mSAn5Wjb1lhnjINLv/GuQ/uSZ0JsSnn9tkA0EBXSo25tu4+iiucvLqjhj/ursNtDCvnTuaey6axZGry2R+wywk9nrcA1e2Gl++BSTPgzj8ETXOJL2igK6X8qt9tONrYzp7qFvbUtLC90snemhbcBhKiI7j7kjzuuiSPKSmx5/YFe1+GV9aB2zWwbvI8+PwfIG6Sb04iSGigK6V8qq/fzd6aFmuu8NLjFJc7T761Z0JkOPNyJnLfVTO4aNokFucmMyHqHJpUTtj9Ivx+HUxZPjBqMywSZq2EmIln/qwNaaArpc6JMYbWbheVxzspbWqntLGD7ZVOtlU46ey1BotPT4vjloVZFE5NZl72RKalxRMe5qPJqna9AK9+GaZeCre9YE2GFeI00JVSZ9TX7+b9kiYOHWujrLGDsuMdHGvppqGtm+4+98n9RKAgPZ5PL8nhovxJLMtPIS0h2j9F7XwOXv07yL8c1r4AUefYXGMzGuhKqVP0uw0dvS6OtXTz8vZqXt5WTVO71Qc8NT6KvElxLMpNIj0hmvSEGKakxJKfGnfuPVLO1vZnYONXYdqVsOZ5DfNBNNCVCjHGGBrbe9hX08remhYO1rdR39JNQ1sPTe09J5tLAMLDhKsvSGfN0ikU5qUwcULk2Bfc1w0n5vvb8yK89jWYfg2seRYizzCpVgjSQFfKxtxuw+GGNraWOthV1UxpUwdlTR20dPWd3GfqpFgyJ8awKDeJ1PhoEmIiiI+OIDEmkitnpZ0+lexYeu+n8OYPBwIdoOA6+OwzEBnAusYpDXSlglxfv5sDda185JlhsMo5MDKysa0HZ6cV3ukJ0cxIj+fm+ZlMS4tnblYic7ISSYgJwF23N955GN76Icy6CXI9r2CIToSFt0GEn9rmg5wGulLjmDEGZ2cfDW3dNLT2UN9qNY00tHZT5eyirKmDKkfnyTftTEuNoyAjnjDPyMhFU5JZmp/CRfkp597P29e8mb/vnZ/A2/8K89fArb+EsDFom7cBDXSlxhm32/BxuYONu2p5fU/dyTvswRJiIshOmsDszARunDeZWZMTWZ6fQnogm0dG09cNr9wLBzaOvi9YL5RY9V8a5mdBA12pcaCjx8UHJU28ebCBNw820NDWw4TIcFbMyWBxbhLpCTGkJ0af7FlyXoNxAqGvG164HUr+B5beO/pw/ITJsOhOCPPBFLghRANdqTHk6ndT0tjO3ppW9tW2cLSxg7KmdmqcXSeHwl8+M5WVczNZMTvduzfqjHd9XbDhNjj6JtzyCCy5O9AV2ZYN/rYoNX7VNnfxfknTyXlMDtS10uOyBuPERoUzLS2OhVOS+eSiHJbnp1CYl0JUhI3uSns7rTAvfRtW/QIWfz7QFdmaBrpSPtLd10/58Q7KGjvYW9vCmwcbOVDXCkB8dARzshK5Y/lU5mVPZG52IvmpPhwGPx71dlhT2Ja9Zz3YXHjamyuVj2mgKzWK2uYuyps66Ojtp6PHRXNnr9XT5MQfT88Tx6A36oSHCUumJvPtGy7gqgvSmZEWT5idw3uo3g547nNQ8QF88tew4HOBrigkaKAr5eF2G+rbuilr7KC0qYNdVc1sLXOcfHvOYBFhQlqC9ZAyJzmWxVOTmZwYQ35qHNPS4shPjbNH+7e3Oh3wuzvh2B5rub8PXF3wyUdh/mcCW1sICaG/cUpZoe3o7OXQsTb21LSwt6aFamcXjW09NLR109c/0Ec6KTaSZXkp3HVJHrMzE0iIjiQuOpzECZGkxEaF1h33mXQch6dXQ9NhawrbMM9ApZnXwYwVga0txGigq6DR0tV3SvPGiejtdbmpdHRS1tRBpaOTfrf7tM+63eDs7KWxrefkIByA7KQJTEuz7qozEmPISprA9NQ48tPiyEiI0dAeTUcTPLUKHEdh7fMw45pAVxTSNNDVuNXrcrOrupk3DjTw5sF6Dte3j7hvmEBOciy5KbFEj9BLZHZmIhmevtzT0+OZmzWR5Lgof5VvT81V8NIXoaXaWu5th/5eWLsBpl8V2NqUd4EuIiuBR4Bw4DFjzI+H2eezwPcBA+wyxugjbeWVfrehxtlFaVM7ZU0dHK5vY29NK4eOtdHb7yYiTFiWn8LqhdnkJE8gPSGGSfFRJ4e3R4QJmUkxREcE2WCbYOOsgKduhu4WmH0LICBhVu+V3OWBrk7hRaCLSDiwHrgWqAaKRGSjMWb/oH0KgG8DlxpjnCKS7q+CVfAyxlDt7GJfrdUn+3C9FeCVxzvp7R9oJpk4IZK52Yl84bI8FuYkcWlBKonjdQKpUOEshydvtl7EfOcfIGtRoCtSw/DmDn0ZUGKMKQUQkQ3AamD/oH3uBdYbY5wAxpgGXxeqgktfv5um9h6OtXSzp6aFraUOtpY5aGrvAay76vzUOKanxbFidgb5qbFMS4snPzWOSXFRiGjb9bjhKIUnb7GaV+78A2QtDHRFagTeBHo2UDVouRq4aMg+MwFE5AOsZpnvG2P+NPRAIrIOWAeQm5t7LvWqcebEA8nSxnb211kvTNhX28qx1u5TJtXLmhjD5QWpLPa8W/KCyQlj83YbdX6OH7XuzF3dcNdrkDk/0BWpM/Am0Ie7VRo6/2UEUAB8AsgB3hORucaY5lM+ZMyjwKMAhYWFXsyhqQKpo8fF/jqrLbulq4+OHhftPS6a2ntoaO3hWGs3tc3WHCRgvVNyelo8F+WnkJcaZ00olRDNrMkJ5CRP0LvuYNN0BJ66xXroeddrMHluoCtSo/Am0KuBKYOWc4DaYfb5yBjTB5SJyCGsgC/ySZXK5068hqzG2UVHTz/tPS5aunopa+qkrKmdkoZ2Sps6TrnLDg8T4qLCSfUMqFmcm8wnF2WTn2oNpJmZkUBctHacGlPGwNv/Bgc3+/7YLZVWn/K7NkHGHN8fX/mcN//1FQEFIpIP1ABrgKE9WF4F1gJPikgqVhNMqS8LVeevqb2H//9RBW8caKCsqYP2Htdp+0SGC1MnxTEtLZ5bFmQxL3siszMTSYmLIjoiTO+yxxO3G17/JhQ9BlMvhZgk3x4/bSZc8Q+QfoFvj6v8ZtRAN8a4ROQ+YAtW+/jjxph9IvIgUGyM2ejZdp2I7Af6gW8aY477s3A1ur5+N9XOLsqa2vnzvnpe2VFDr8vNRfkpfHpJDvmpceQkTyAhxjMCMiaSzIkxRITbaLY/u3K74Y9/D9uegEvuh2sftNq8VEgT483roPygsLDQFBcXB+S77cbV7+bP++t54oMyjjQMDL5p73adHBUZExnGpxbn8MXL8pmeFh+oUtVwOppg0wPQdsz7z/S2Q8N+uOwBuOafNcxDiIhsM8YUDrdNGzyDmKOjl5e3VfPUh+VUO7uYOimWW+ZncWK0enxMBPmp8Z727fjx+zLgUNbeYA2dd5af3eCcqDjrFW0Xf0XDXJ2kgR5kjDF8WHqc57ZWsmXfMfr6DcvyUvjuzXNYMTvD3vNr201bvdWLpKUKbn8R8i8PdEUqyGmgB4lel5vXdtXy2PtlHKhrZeKESO5YPpW1y3KZmZEQ6PLGh+Yq+PN3oKt59H3Hg+MlVq23vwR5lwa6GmUDGujjnLOjl+c+ruSpv5bT0NbDzIx4HvrUPFYvzNaBOYOdmGek0wkZFwa6Gu9MmgFX/ZPOg6J8RgN9nKpv7eYXb5bw4rYquvvcXDEzjYc/k88VBanadXAoR5nVdNHTCndthOzFga5IqYDQQB9n2rr7+PU7pTz2fin9bsMnF2Vzz2XTmDU5SJpV+vvgzR9A4+Gx+87aHdbQ9Ds36jwjKqRpoI8TvS43z26t4L/eLMHR0cstC7L45nWzyJ0UG+jSvOfqhZe+AAc3QcZca2rVsTBpOqz8sc4zokKeBnqA9bsNm/fU8fCWQ1Q6Orl42iS+feMFzM/x8ag/f3P1wIt3w6HNsPIhWP7lQFekVMjRQA+Qjh4XL22r5vEPyqg43skFkxN48gtLuXJmWmDayOv3w1//0wrmoaJirSHgyVOH/6yrx3pB8OE/wY3/Dsvu9W+tSqlhaaCPsZbOPh7/oIwnPiijtdvFotwkvnn9LG6Ymxm4PuR1u+HpVeDuh/iM07e31kDpO3D3JkjOO3VbXzf87vNw5M9w03/A0i+NSclKqdNpoI+Rrt5+fvl2CU9+UE5bj4vr5mTwf66czpKpyYEtrHan9cb2qHi4+zVImTbyPk/cZIV6Sr61vq8bXrgdSv4Hbv45FH5hbGtXSp1CA30MdPa6+OKTRXxU6uCGuZO5/5oCZmcmBrosq5nl6VUQPdEK86F33ydkLbS6Az69Gp64EaZfba1v2G/1MFn1X7D4zjErWyk1PA10P2vvcfHFJ4oornDwyJqFrF6YHeiSBmz5JwiL8DSljNA+fkLmAuslB6/+LZS+ba0Lj4BbfwUL1/q9VKXU6DTQ/aitu48vPFHEjqpmHlmziFsWZAW6pAEVH0LpW3Ddj0YP8xMmz4Mvv+/fupRS50wD3U+cHb3c/cTH7K1t5T/XLOKm+ZljW0DJG3Do9YHljDmw5AsDM/O9/a8Qlw6FXxzbupRSfqOB7gf1rd18/rdbKT/eyX/fsYRr5wzTc8Sf9rwEr6yDyAkQHgWmH4paoKkErv8RVHwAZe9ag3GigmjgklLqjDTQfaza2cltv9lKU3sPT969lEtmpI5tAbtfhN+vg9yL4bbfQXS89d7J1/8RPloPxg3H9kD8ZFhy99jWppTyKw10H+rq7edLTxXT3NnLs1+6iEW5Y9wlcdcL8OqXrfdL3vaC9RIEsJpZbngIwsLho19a62542LqDV0rZhga6jxhj+M6rezhU38bjdy8d+zDf+Ry8+nfWSxLWvnB6U4oIXP+vEBkL5e9pN0OlbEgD3Uee3VrJK9tr+PqKAq6alT62X779Gdj4VZh2Jax5fuR2cRG45rtjW5tSasxooPvAzqpmHnxtP5+Ylcb9Vxf4/wv3/8EaFATQ5YCPH7UG+6x5TptRlAphXgW6iKwEHgHCgceMMT8esv1u4GGgxrPqF8aYx3xY57jV0ePiaxt2kJYQzc8/t5Awf8/H8t5/wBsPnrrugpvhU7+FyBj/frdSalwbNdBFJBxYD1wLVANFIrLRGLN/yK4vGGPu80ON49oPNu2n0tHJC+suJik2yr9f9s7D8NYPYd5n4Nb/tkZqKqWUhzeJsAwoMcaUAojIBmA1MDTQQ85f9tezoaiKv/3EdJblp5z/Ad39sOt56Gg6fZujFLY/BfPXwK2/tHqsKKXUIN4EejZQNWi5GrhomP0+JSJXAIeBB4wxVUN3EJF1wDqA3Nzcs692HGls6+FbL+9mTmYiD6yYef4H7HdZ86Ts+d3I+yy+05rVUMNcKTUMbwJ9uEZhM2T5NeB5Y0yPiHwZeAq4+rQPGfMo8ChAYWHh0GMElX95bR9tPS6eX7OQqIjzfNVav8saDLT3Zbj6u7D8707fR0QfeCqlzsibQK8GpgxazgFqB+9gjDk+aPE3wEPnX9r4taPSyabdddx/9QxmZpzny5v7++CVe2Hf72HFv8BlX/dNkUqpkOPNrWURUCAi+SISBawBNg7eQUQGzzy1CjjguxLHF2MM//b6QVLjo1h35fTzO1h/H7z0RSvMr/2BhrlS6ryMeodujHGJyH3AFqxui48bY/aJyINAsTFmI3C/iKwCXIADuNuPNQfUGwca+LjMwQ9unUt89Hn0MnH1wktfgIObrBGcF3/Fd0UqpUKSGBOYpuzCwkJTXFwckO8+V65+NysfeQ+327DlgSuIDB/lHzjdrXDwj+DuO33bgU1wZAusfAiWf9k/BSulbEdEthljCofbph2Zz8KL26opaWjnv+9YMnqYdzrgmVuhbtcIOwjc+O+w7F6f16mUCk0a6F4yxvCbd0tZMCWJ6y8cZX7zTof1rs7Gw/DZZyB78en7RMZCrA/6riullIcGupd2VjVT2tTBTz41H5EzDO/vOG6F+fESWPsczFgxdkUqpUKaBrqXXtleQ3REGDfMm3zmHbf8EzQdgds2WBNmKaXUGDnPETGhocfVz2u7a7n+wskkxESOvGPTEWuk57J7NcyVUmNOA90Lbx1spLmzj79ZnH3mHd95CCJi4FLtT66UGnsa6F54ZXs1aQnRXHam94M2HrJezrzsXohPG7vilFLKQwN9FI6OXt461MCtC7OIOFNXxbd/bL3D85KvjV1xSik1iD4UHcVru2rp6zd8aknOqRu6nFC7w/q902EN37/sAYibNPZFKqUUGuijenl7NbMzE7lgcuKpGzZ+FQ68NrAcnQiXfHVsi1NKqUE00M/g4LFWdle38L2b55y6obsVDm+BBWthyd3WusQsHSiklAooDfQzeLG4mshw4dZFQ3q3HNoM/b1QeA9MWRqY4pRSagh9KDqCXpeb3++o4do5GaTEDXlX6L7fw8QpkDPs/DhKKRUQGugjePNgA46OXj5TOOXUDV3NUPIGXHir9RYhpZQaJzTQR/BicRUZidFcUTCkT/mJ6XAv/GRgClNKqRFooA+jvrWbtw418KnFOYSHDbkL3/d7SJoKWcPMoKiUUgGkgT6MV7bX4Dac3tzS6YDSt6y7c21uUUqNMxrow3h1Rw1L85LJT407dcPBTeB2aXOLUmpc0kAf4nh7D4fq27j6giEvsWg7Bu//HFKmQeaCwBSnlFJnoP3QhygqdwKwLD95YGVrLTx1ixXqd7ykzS1KqXFJA32IonIH0RFhzMtOsla01MBTN0N7A3z+FchdHtgClVJqBBroQxSVO1g4JYmoCE9r1Kt/C+2NcMcrkHtRYItTSqkz8KoNXURWisghESkRkW+dYb9Pi4gRkaAcQtnR42JfbSvL8j1zsvT3QdVWWHynhrlSatwbNdBFJBxYD9wAzAHWisicYfZLAO4Htvq6yLGyvdJJv9uwNM8T6I0HwdUN2drnXCk1/nlzh74MKDHGlBpjeoENwOph9vsB8BOg24f1jamiMgdhAotyPe3nJ+Y7z1wYuKKUUspL3gR6NlA1aLnas+4kEVkETDHGbDrTgURknYgUi0hxY2PjWRfrbx+XO5iTlTjwIujandY85ynTAluYUkp5wZtAH66Pnjm5USQM+BnwjdEOZIx51BhTaIwpTEsbX+/d7HW52VHZPNDcAtYdeuYCCNPu+kqp8c+bpKoGBo+BzwFqBy0nAHOBt0WkHFgObAy2B6N7alrocblZdiLQXb1QvxeyFgW2MKWU8pI3gV4EFIhIvohEAWuAjSc2GmNajDGpxpg8Y0we8BGwyhhT7JeK/aSo3AFA4YlAb9hvvcQiS9vPlVLBYdRAN8a4gPuALcAB4HfGmH0i8qCIrPJ3gWOlqMzBtNQ40hKirRV1O62feoeulAoSXg0sMsZsBjYPWfe9Efb9xPmXNbbcbkNxhZPrLxw0f0vtDoiZCMn5gStMKaXOgj7tA0qbOmjp6mPJ1EHzt9TusO7Odd4WpVSQ0EAHdlRaE3ItzvUEel831O/X/udKqaCigQ7sqGomISaC6Wnx1oqGfdZr5rT9XCkVRDTQge0VThZOSSLsxOvmavWBqFIq+IR8oLf3uDhc38ai3CHt5xNSICk3cIUppdRZCvlA313djNsMmr+lqxnK3rH6n+sDUaVUEAn5QN9R2QzAoilJ0OWEZ26F1jq4+L4AV6aUUmcn5F9wsaPSybS0OJJoh6dvhYYDsOZZmHFNoEtTSqmzEtJ36MYYdlQ2W90VN9wGDQdhzXMw8/pAl6aUUmctpAO9ytHF8Y5elmWGQ+WHcPk3oODaQJellFLnJKQDfbtnQNHS2AZrReaCAFajlFLnJ6QDfUelk9iocKa6K60V6RcEtiCllDoPoR3oVc0syEkirPEgRMbCRO13rpQKXiEb6F29/eyvbbX6nzcegLRZ+mYipVRQC9kE21HlxOU21ivnGg5C2uxAl6SUUuclZAO9qMyJCCxJB9qPafu5UirohW6glzu4YHIiiW0l1gq9Q1dKBbmQDHRXv5vtlU6W5SVbI0MB0jXQlVLBLSQDfV9tK529/SzNT4HGgxCVABNzAl2WUkqdl5AM9KJyB4Dngainh4vOrKiUCnIhGegflznITYklIzHGukPXB6JKKRvwKtBFZKWIHBKREhH51jDbvywie0Rkp4i8LyJzfF+qbxhjKK5wWnfnHU3Q0agPRJVStjBqoItIOLAeuAGYA6wdJrCfM8bMM8YsBH4C/NTnlfrI0cZ2HB29LMsf/EBU79CVUsHPmzv0ZUCJMabUGNMLbABWD97BGNM6aDEOML4r0bc+LvNMyJXneSAKeoeulLIFbwI9G6gatFztWXcKEfmKiBzFukO/f7gDicg6ESkWkeLGxsZzqfe8FZU7SI2PIj81zrpDj06ExKyA1KKUUr7kTaAP1/3jtDtwY8x6Y8x04B+B/zfcgYwxjxpjCo0xhWlpaWdXqY98XOZgaV4KIuJ5IDpbe7gopWzBm0CvBqYMWs4Bas+w/wbg1vMpyl+qHJ3UNHdZzS3GeLosavu5UsoevAn0IqBARPJFJApYA2wcvIOIFAxavAk44rsSfefdI1YzzxUzU6G9AbocOkJUKWUbo74k2hjjEpH7gC1AOPC4MWafiDwIFBtjNgL3icgKoA9wAnf5s+hz9e7hRrKTJjA9LR7Ktlkr9Q5dKWUTowY6gDFmM7B5yLrvDfr9az6uy+f6+t18UHKcWxZkWe3nDZ4eLnqHrpSyiZAZKbq9wkl7j4srZ3oexjYegJgkiM8IbGFKKeUjIRPo7xxuJCJMuGTGJGtFw0FIn6M9XJRSthFSgb54ajKJMZFWD5fGAzpCVCllKyER6A1t3eyrbR1obmmrg+4WHSGqlLKVkAj09w43AQwEus7hopSyoZAI9HcON5IaH82czERrhc7hopSyIdsHer/b8N6RRq6YmUpYmOcBaMMBiE2F+MBMP6CUUv5g+0A/UNeKs7OPKwoGhfeJOVyUUspGbB/oJ143d9G0FGuFMVaXRR0hqpSyGdsHenG5k+ykCWROnGCtaKmG3jZ9IPAJJ0IAAArRSURBVKqUsh1bB7r1ujkHhXnJAyv1gahSyqZsHejVzi7qW3sonDoo0E92WdRAV0rZi60DvbjCaj8vzEsZWNl40Jq/JTZlhE8ppVRwsnWgF5U7SYiOYGZGwsDKhv36QFQpZUu2DvTicgeLpyYTfqL/udsNjYe0uUUpZUu2DfSWzj4O17ef2n7uLIO+Tg10pZQt2TbQt1UO035et9P6mbUoABUppZR/2TbQi8udRIQJC6ckDays3QHhUdplUSllS7YO9AuzJzIhKnxgZe1OyJgLEVGBK0wppfzEloHe4+pnV3UzSwe3n7vdULdLm1uUUrZly0DfW9NKj8t96ghRRyn0tGqgK6Vsy6tAF5GVInJIREpE5FvDbP97EdkvIrtF5A0Rmer7Ur23vcIJwOLBd+i1O6yfGuhKKZsaNdBFJBxYD9wAzAHWisicIbvtAAqNMfOBl4Cf+LrQs7GtwkluSizpCTEDK2t3QESMDipSStmWN3foy4ASY0ypMaYX2ACsHryDMeYtY0ynZ/EjIMe3ZXrPGMO2SidLBt+dg9VlcfI8CI8ITGFKKeVn3gR6NlA1aLnas24k9wCvD7dBRNaJSLGIFDc2Nnpf5VmodnbR2NZzaqC7+/WBqFLK9rwJdBlmnRl2R5E7gELg4eG2G2MeNcYUGmMK09L88/q3bZ7281MC/XgJ9LZroCulbM2b9odqYMqg5RygduhOIrIC+A5wpTGmxzflnb3iCgfxQyfk0geiSqkQ4M0dehFQICL5IhIFrAE2Dt5BRBYBvwZWGWMafF+m97ZVNLMoN2lgQi6wBhRFxkLqzMAVppRSfjZqoBtjXMB9wBbgAPA7Y8w+EXlQRFZ5dnsYiAdeFJGdIrJxhMP5VVt3H4eOtbI4d8gD0dodkLkAwsKH/6BSStmAV10+jDGbgc1D1n1v0O8rfFzXOdlV1YLbcOqAon4XHNsNi+8KXGFKKTUGbDVSdFuFExFOnZCr8YA1Za62nyulbM5WgV5c4WBWRgIJMZEDK8vft35OvSQwRSml1BixTaD3uw07K5tPH1BU/j4k50HSlGE/p5RSdmGbQD/S0EZbj2vIgCK3Feh5lweuMKWUGiO2CfTi8mEGFNXvge5mDXSlVEiwTaBvq3CSGh9NbkrswMqy96yf+RroSin7s02gF1c4KJyajMigAUXl70PKdEjMClxhSik1RmwR6A2t3VQ5uk7tf+7uh4q/6t25Uipk2CLQh52Qq24X9LRo+7lSKmTYItCLK5xER4RxYdbEgZXlnvbzvMsCU5RSSo0x2wT6gpwkoiIGnU75+9ZkXAmTA1eYUkqNoaAP9K7efvbVtLBk6PwtFR9qc4tSKqQEfaDvqm7G5TYUDm4/3/E09LbpA1GlVEgJ+kA/7YHotqdg0wMw/WqYdWMAK1NKqbEV9IFeXO5gRno8SbFRUPw4vHY/zFgBa56HiOhAl6eUUmMmqAPd7TZsq3BazS1VH1t35gXXweeehciYQJenlFJjKqgD/WhjO63dngm5St+xVv7NbzTMlVIhKagD/cPS4wAU5qVA3U6YNAMmJI3yKaWUsqegDvQ/7q5jRno8eZNirfeG6luJlFIhLGgDvaG1m4/LHdw0LxNpb4DWGg10pVRIC9pAf33vMYyBm+dnWs0toIGulAppQRvom3bXMisjgYKMBKu5BYHJ8wNdllJKBYxXgS4iK0XkkIiUiMi3htl+hYhsFxGXiHza92We6lhLN0XlTm6an2mtqN1pzdsSHe/vr1ZKqXFr1EAXkXBgPXADMAdYKyJzhuxWCdwNPOfrAoezeU8dwKBA1weiSikV4cU+y4ASY0wpgIhsAFYD+0/sYIwp92xz+6HG0/xxTx2zMxOZnhYPrXXQfkwDXSkV8rxpcskGqgYtV3vWnTURWScixSJS3NjYeC6HoLa5i20VTuthKHjaz9FAV0qFPG8CXYZZZ87ly4wxjxpjCo0xhWlpaedyiIHmlnmDAl3CYPK8czqeUkrZhTdNLtXAlEHLOUCtf8oZ3SdmpRMmQl5qnLWibiekzYao2ECVpJRS44I3gV4EFIhIPlADrAFu82tVZzAjPZ4Z6Z7eLMZYd+gF1wWqHKWUGjdGbXIxxriA+4AtwAHgd8aYfSLyoIisAhCRpSJSDXwG+LWI7PNn0Se11kBHo7afK6UU3t2hY4zZDGwesu57g34vwmqK8b/tz8CHv7B+7+u0fmqgK6WUd4E+rsSmQNqsgeWC6yBzQeDqUUqpcSL4Av2Cm6w/SimlThG0c7kopZQ6lQa6UkrZhAa6UkrZhAa6UkrZhAa6UkrZhAa6UkrZhAa6UkrZhAa6UkrZhBhzTjPhnv8XizQCFef48VSgyYflBItQPO9QPGcIzfMOxXOGsz/vqcaYYecfD1ignw8RKTbGFAa6jrEWiucdiucMoXneoXjO4Nvz1iYXpZSyCQ10pZSyiWAN9EcDXUCAhOJ5h+I5Q2iedyieM/jwvIOyDV0ppdTpgvUOXSml1BAa6EopZRNBF+gislJEDolIiYh8K9D1+IOITBGRt0TkgIjsE5GvedaniMhfROSI52dyoGv1NREJF5EdIrLJs5wvIls95/yCiEQFukZfE5EkEXlJRA56rvnFIXKtH/D8/d4rIs+LSIzdrreIPC4iDSKyd9C6Ya+tWP7Tk227RWTx2X5fUAW6iIQD64EbgDnAWhGZE9iq/MIFfMMYMxtYDnzFc57fAt4wxhQAb3iW7eZrWC8jP+Eh4Geec3YC9wSkKv96BPiTMeYCYAHW+dv6WotINnA/UGiMmQuEA2uw3/V+Elg5ZN1I1/YGoMDzZx3wq7P9sqAKdGAZUGKMKTXG9AIbgNUBrsnnjDF1xpjtnt/bsP4Dz8Y616c8uz0F3BqYCv1DRHKAm4DHPMsCXA285NnFjuecCFwB/BbAGNNrjGnG5tfaIwKYICIRQCxQh82utzHmXcAxZPVI13Y18LSxfAQkiUjm2XxfsAV6NlA1aLnas862RCQPWARsBTKMMXVghT6QHrjK/OLnwD8Abs/yJKDZGOPyLNvxek8DGoEnPE1Nj4lIHDa/1saYGuDfgUqsIG8BtmH/6w0jX9vzzrdgC3QZZp1t+12KSDzwMvB1Y0xroOvxJxG5GWgwxmwbvHqYXe12vSOAxcCvjDGLgA5s1rwyHE+78WogH8gC4rCaHIay2/U+k/P++x5sgV4NTBm0nAPUBqgWvxKRSKwwf9YY84pndf2Jf4J5fjYEqj4/uBRYJSLlWE1pV2PdsSd5/kkO9rze1UC1MWarZ/klrIC387UGWAGUGWMajTF9wCvAJdj/esPI1/a88y3YAr0IKPA8CY/CeoiyMcA1+Zyn7fi3wAFjzE8HbdoI3OX5/S7gD2Ndm78YY75tjMkxxuRhXdc3jTG3A28Bn/bsZqtzBjDGHAOqRGSWZ9U1wH5sfK09KoHlIhLr+ft+4rxtfb09Rrq2G4E7Pb1dlgMtJ5pmvGaMCao/wI3AYeAo8J1A1+Onc7wM659au4Gdnj83YrUpvwEc8fxMCXStfjr/TwCbPL9PAz4GSoAXgehA1+eH810IFHuu96tAcihca+BfgIPAXuAZINpu1xt4HusZQR/WHfg9I11brCaX9Z5s24PVA+isvk+H/iullE0EW5OLUkqpEWigK6WUTWigK6WUTWigK6WUTWigK6WUTWigK6WUTWigK6WUTfwvd6vQxxWXl28AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores3)\n",
    "plt.plot(test_scores3)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ReLU VS Sigmoid VS Tanh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3gVVfrA8e/JTSWBQAihBQi9SwuhKSBIkSqKBUVlRVBXUGH1Z1sLKoqK61qwILJgASxYUBEURECkpNISSoBAQoBU0ttNzu+PGzEkAS7h9ryf5+EhM3PuzDsZ8nJy5hSltUYIIYTzc7N3AEIIISxDEroQQrgISehCCOEiJKELIYSLkIQuhBAuwt1eFw4MDNQhISH2urwQQjilyMjINK11o+qO2S2hh4SEEBERYa/LCyGEU1JKHb/QMWlyEUIIFyEJXQghXIQkdCGEcBF2a0OvTklJCUlJSRQWFto7FIfn7e1NcHAwHh4e9g5FCOEgHCqhJyUlUbduXUJCQlBK2Tsch6W1Jj09naSkJFq3bm3vcIQQDsKhmlwKCwtp2LChJPNLUErRsGFD+U1GCHEeh0rogCRzM8n3SQhRmUM1uQghhKspKC7l853HyS4oObdveOfG9GhR3+LXkoReicFgoHv37hiNRlq3bs2nn35K/foX/8b7+fmRm5t73r5p06Yxbtw4Jk+efNFyQgjXlZJdyIxPItidlEXFX6qD6nlLQrcFHx8fYmJiALj77rtZtGgRTz/9tJ2jEkI4m9jkbO5dHs7ZghI+uiuUEV0aW/2aktAvYsCAAezZs+fc9uuvv86XX35JUVERkyZNYt68eXaMTghhb5/uOM77m+IpLq268ltWQTENfb346v4BdG3mb5N4HDahz/thP7HJ2RY9Z5dm9XhufFezypaWlrJx40amT58OwC+//MLhw4fZtWsXWmsmTJjAli1bGDx4sEVjFEI4PmNpGS/9FMeyPxMIax1AuyC/KmXqeBiYMbgNjet52ywuh03o9lJQUEDPnj1JSEigT58+jBgxAjAl9F9++YVevXoBkJuby+HDhy+Y0KvrhSI9U4RwfjmFJcxeGc3vB1OZfnVrnhrTGYObY/xsO2xCN7cmbWl/taFnZWUxbtw4Fi1axEMPPYTWmieffJL77rvPrPM0bNiQzMzMc9sZGRkEBgZaK2whhA0kZuRz7/II4lNzmT+pG3f0a2XvkM7jcP3QHYW/vz9vv/02CxcupKSkhFGjRrF06dJzvVROnjxJSkrKBT8/dOhQvvjiC4qLiwFYtmwZ1157rU1iF0JYXtSJTCa9t43krAKW/yPM4ZI5mFlDV0qNBt4CDMASrfWCSsdbAsuB+uVlntBar7VwrDbXq1cvevTowapVq7jzzjuJi4tjwIABgKkL4meffUZQUBD5+fkEBwef+9zcuXOZO3cukZGR9OnTB4PBQNu2bfnggw/sdStCiAv4MiKRdftOX7SM1pptR9Jp6u/Nqpl9/24zP74dtr8LpSUX/XwVfe+FDiNrGPGFKa2rvp09r4BSBuAQMAJIAsKBKVrr2AplFgPRWuv3lVJdgLVa65CLnTc0NFRXXuAiLi6Ozp071+Q+aiX5fglxZQ6ezmHM21tp6u9NgzqeFy3bsmEdXpzYjQDf8nIxK2HNbKgTAHWbXt6Fr5kLXSbWKGalVKTWOrS6Y+bU0MOAeK310fKTrQImArEVymigXvnX/kByjSIVQggb0Vrz3Jp91PV254dZV9PA9+IJ/ZyyMtj0Emx9A1oPhls+AZ8G1g3WTOYk9OZAYoXtJKBfpTLPA78opWYDvsB1FolOCCGs5Mc9p9hxNIOXbuhmfjIvzofv7ofY76H33TD2DTA4zhTW5rwUra4/TuV2minAMq11MDAG+FQpVeXcSqmZSqkIpVREamrq5UcrhBAWkF9s5OW1cXRtVo8pYS3N+1DOaVg2FmLXwMiXYPxbDpXMwbwaehLQosJ2MFWbVKYDowG01tuVUt5AIHBeNxCt9WJgMZja0GsYsxBCXB6tYd9qSDsMQGR8KrfmZXJLpxYYNv9hzgkg+nMoyITbVkCnMdaNt4bMSejhQHulVGvgJHAbcHulMieA4cAypVRnwBuQKrgQwv5KS+CnuRD1ybld1wDXuAO7L+M8DULgnnXQ9CoLB2g5l0zoWmujUmoWsB5Tl8SlWuv9SqkXgAit9RrgX8BHSqk5mJpjpulLdZ8RQghrK8iEL++CY1soHjiXB5Ov59cDqdw1oBXPjuuCu8G1huKY1Q+9vE/52kr7nq3wdSwwyLKh2c/8+fNZsWIFBoMBNzc3PvzwQz766CPmzp1Lly5drHbdMWPGsGLFiirT9T7//PP4+fnx6KOPWu3aQlyuxIx85nwRw/GM/GqPD+nQiJdu6Ia3hwGAImMpz6+J5XRWAQtv7kFDPy8Ayso0b/92mJW7TlBmwWpgC53Mm8ZXaEYKLxlm8822/uQVp/L8+C5MG+SaSzc67NB/e9m+fTs//vgjUVFReHl5kZaWRnFxMUuWLLH6tdeudfqxWKKWiDyewcxPIikpLWPsVU2p3Hciv9jI6qgkDqfk8tFdfXB3c+O+TyMIT8jE0+DGpPf+ZOm0UIIb1OFfX+3mpz2nGNqxEU39fSwSX+vcaKYefxptUCwNeYti3x6MVzC2e1MGtXPdKTgkoVdy6tQpAgMD8fIy1R7+mn9l6NChLFy4kNDQUD7++GNeffVVmjVrRvv27fHy8uLdd99l2rRp+Pj4cODAAY4fP87//vc/li9fzvbt2+nXrx/Lli0DYOXKlbz88storRk7diyvvvoqACEhIURERBAYGMj8+fP55JNPaNGiBY0aNaJPnz52+X6I2klrTXhC5nmr7PzlREY+C9YdoJm/N0un9aVNo6ozDQKM6d6UR1bFcMO723A3uHEmu5B3pvQiuIEPMz6JZNJ7f9IyoA6xp7J58vpOzBzcpmYT2GkNibugIMO0nR4PG+ZBQGu4/QvuC2hz+ed0Uo6b0H9+Ak7vtew5m3SH6xdctMjIkSN54YUX6NChA9dddx233norQ4YMOXc8OTmZF198kaioKOrWrcuwYcPo0aPHueOZmZn89ttvrFmzhvHjx7Nt2zaWLFlC3759iYmJISgoiMcff5zIyEgaNGjAyJEj+e6777jhhhvOnSMyMpJVq1YRHR2N0Wikd+/ektCFzRQZS3nqm32sjkq6YJmw1gF8OLXPRftvj+rahK/uH8D05eEUFxtZNbM/vVqaBuB89+BA7l0ewdHUPD6Y2odRXZvULNjSEvj5/yBi6fn7Ww8pH/Bj+VWBHJnjJnQ78fPzIzIykq1bt7Jp0yZuvfVWFiz4+z+BXbt2MWTIEAICAgC4+eabOXTo0Lnj48ePRylF9+7dady4Md27dwega9euJCQkcPz4cYYOHUqjRo0AuOOOO9iyZct5CX3r1q1MmjSJOnXqADBhwgSr37cQABl5xdz/aSS7EjJ4aHh7RnSuusqOmxt0bFzXrBeK3Zr7s/FfQynTmnref/fZDm5Qh+9nDSKn0EhgeVv6ZSs4C19Ng6ObYNDD0HVSeYDuENQF3Aw1O68Tc9yEfomatDUZDAaGDh3K0KFD6d69O8uXLz937FKdd/5qqnFzczv39V/bRqMRd3fzvuUyd7qwtfiUXKYvD+dUViFvT+nFhB7NLHJeP6/q/817uRvw8ruMpHsmFra8BiUFpu2UOMhOhomLoNdUC0Tq/Fyrz44FHDx4kMOHD5/bjomJoVWrv6fJDAsLY/PmzWRmZmI0Glm9evVlnb9fv35s3ryZtLQ0SktLWbly5XlNOgCDBw/m22+/paCggJycHH744YcruykhLmFbfBo3vreN3EIjK2f0t1gyt5hDv8DHI+Ho75BzyvSnblO481tJ5hU4bg3dTnJzc5k9ezZnz57F3d2ddu3asXjxYiZPngxA8+bNeeqpp+jXrx/NmjWjS5cu+Pubv15g06ZNeeWVV7j22mvRWjNmzBgmTjx/1rXevXtz66230rNnT1q1asU111xj0XsUoqIVO0/wzPf7aNvIl4/v7kuLgDr2DulvWsPOD2H9k9C4K0z5Avyb2zsqh3XJ6XOtxZmnz83NzcXPzw+j0cikSZO45557mDRpks3jcJbvl3BMpWWaV9bGseSPYwzp0Ih3b+9FXW8Hmpuk1AjrHofwJdBxDNz4EXhV36OmNrnS6XNFJc8//zwbNmygsLCQkSNHnvdCUwhnkFdk5OFV0WyIS+HuAa14xtFGTRZmmV54HvkNBs6G6+bVypecl0sSeg0sXLjQ3iEIcVlScgr5KiKJktIyANbvP8OhMzm8MLErdw0IsW9wlWUmwIpbTf3JJ7wDve+yd0ROQxK6EC5Oa83DK2PYfjT93L4AX0+WTuvLkA6N7BhZNU7shFW3Q5nR9MKz9WB7R+RUJKEL4eLW7j3N9qPpvDixK1P7/91jy+G6xu75Cr5/0PTS8/avILCdvSNyOpLQhXBh+cVGXvopli5N63F7v1aOk8RLjbDxedi7GtCm3iy5p6HV1XDrp6Z1OsVlk4QuhAt7b9MRTmWZ5lAxuDlIMi/Mhq/vgfhfTb1XfMsny/JvaRrx6W7mcnCiCknoFaSnpzN8+HAATp8+jcFgODdEf9euXXh6mv8PberUqUyePFl6wAi7SUjLY/GWo0zq1ZzQEDvWeLOT/56XqcwIv70EqQdh7H+g73T7xWUjWmtiUmPIKc45t69d/XY087P84C1J6BU0bNiQmJgYQOYgF87vo61HMbgpnri+k/2COLIJvrwbirL+3uflD1NXQ9tr7ReXjZSUlvDCjhf4Lv678/Y/0/8Zbul4i8WvJwndTOPHjyc5OZnCwkLmzJnDvffei9FoJDAwkPvvv5+ff/6ZOnXq8P333xMUFATApk2beO211zh9+jRvvPGGXQYfidpJa83GuBSGdGhE43re9gki4n/w078gsAOMXQEe5XOd1w8B34b2icmGzhaeZc7vc4g4E8GM7jMY1nLYuWNNfZta5ZpmJXSl1GjgLUxL0C3RWi+odPxN4K//busAQVrrK5q38tVdr3Ig48CVnKKKTgGdeDzs8Rp9dvny5QQEBJCfn09oaCg33XQTdevWJSsriyFDhrBgwQLmzp3L0qVLeeKJJwBISUlh27Zt7N27l1tuuUUSurCZ2FPZnM4uZFjnIOtdpDgfNs2HFv2gS4UZQctK4ddnYfu70G4ETF4K3vWsF4cdxaTE8EnsJ5SUVp03/lDmIVILUnnlmlcY12acTeK5ZEJXShmARcAIIAkIV0qtKV92DgCt9ZwK5WcDvawQq129+eabrFmzBoCkpCSOHDlCz5498fHx4frrrwegT58+bN269dxnbrjhBpRSXHXVVZw8edIucYvaadOBFACu7WilhJ5zGlbeBsnRpsR97dMw+DEozoPV98KhnyHsPhj1MhhcsyHghyM/8Nyfz1HPsx5Bdap+n5v4NmHB4AX0CrJdOjTnOx0GxGutjwIopVYBE4HYC5SfAjx3pYHVtCZtDRs2bGDLli3s2LEDHx8frr76agoLCwHOe1FqMBgwGo3ntitOnytrZgtb2ngghR7B/jSqW8O5xi/m1B5TMi84a1pE4sBPppp66gFIPQQpsTBmIYTNsPy17SijMIMybRppu/LAShbvWUxYkzD+M/Q/+HuZP0GfNZmT0JsDiRW2k4B+1RVUSrUCWgO/XXlojiMrK4uAgAB8fHzYv38/4eHh9g5JiAtKzy0iJvEsjwzvYPmTZyfDsrHgVRemrzetAtZ5AgS2N/Ve8aoHd3wJ7a6z/LXt6L+R/+XjfR+ft+/G9jfy737/xsPgOBOamZPQq+u8eqHq5m3A11rr0mpPpNRMYCZAy5YtzQrQEYwdO5bFixfTo0cPOnXqRL9+1f5/JoRD+P1gKlrDcGu0n//yDBiL4L7N8NdanUqZmltaDYK6Tf7e7yIOZR5i2f5lDGsxjEHNBwEQVCeIIcFDHGegVrlLTp+rlBoAPK+1HlW+/SSA1vqVaspGAw9qrf+81IWdefpcRyHfL1GdBz+PIjwhg51PDbdswknYBsvGwJDH4dqnLHdeB6a15p719xB/Np4fJ/3oEE0rF5s+15z5MsOB9kqp1kopT0y18DXVXKQj0ADYfiXBCiFqrthYxpZDqQzrFGTZZF5qhLWPgX8LGPSI5c7r4NYlrCPiTAQP9X7IIZL5pVwyoWutjcAsYD0QB3yptd6vlHpBKVVx9eIpwCotb/+EsJuIhAxyiowM62Th5paIpZCyH0bNB08HWtHIivJL8lkYsZDOAZ25sd2N9g7HLGb1J9JarwXWVtr3bKXt5y0RkNba4dqlHJH8vykACktKefb7fWw5lAZAXrERT4Mbg9oFWu4iydGmF55thppegDqJxOxEnvrjKZLzkmv0+eLSYs4WneWNIW9gcJLFNRyqg6i3tzfp6ek0bNhQkvpFaK1JT0/H29tOIwCFQ0jNKWLGJxHsTjrL2O5N8fU0/Tj3bFkfXy8L/WjHroFvZoJvIxj3pukFqBOIPBPJI5seQaMZ1mJYjfNJr6Be9AzqaeHorMehEnpwcDBJSUmkpqbaOxSH5+3tTXBwsL3DEHZy8HQO9ywLJz2viPfv6MPobk0sewGtYdtbsOE5aB4KU1aCnxVHnV4BrTV70vaQVT5fzPHs4/wn8j8E+wWzaPgiWtZznh51V8qhErqHhwetW7e2dxhCODStNf/8PJKS0jK+um8g3YOt8LIu7gdTMu96I9zw3t/zsDgYY5mRBbsW8MXBL87b369JP94Y+oZTvMi0JIdK6EKIS9t1LIMjqXm8Pvkq6yTz4nxY/xQ07gY3fuSwQ/dzinN4dPOj/Jn8J3d3uZtRIaMAMLgZ6Nigo9O0e1uSYz4pIcQFrQpPpK6XO2Ovss6MfWz7L2QlwqQPHTaZJ+UkMWvjLI5nH2fewHnc2N45eqFYm2M+LSFEtbLyS1i79xST+wRTx9MKP74Zx+CP/0K3yRAyyPLnt4DolGge/u1hSnUpH474kLCmYfYOyWFIQhfCiXwXc5IiYxlTwqz0om/90+DmDiNftM75ayCrKIuSMtP0tNtObmPe9nk09W3KouGLCPEPsW9wDkYSuhBOQmvNyl0n6Na8Ht2aW7jtXGvYuhAO/gTXPQ/1LL882uUylhl5Pfx1VhxYcd7+0MahvDn0Tep7X9GSCy5JEroQTmJ3UhYHTufw0g3dLHtiYxH88DDsXgndb4YBsyx7/hrILc7lsS2P8cfJP7ip/U10DjDNWVTHow6jQ0Y71AyHjkQSuhAOSmvNT3tPcTQ1D4Bt8Wn4eBiY2NOCtee8dPhiKpz48+9FKqw4eGhf2j7+OPnHJcutT1hPQlYCzw54lps73Gy1eFyNJHQhHFBJaRnPfr+flbtOnLf/nkGtqettodpp6kFYcQtknzItE9ftJsuc9wK+i/+OedvnYSwzXrJsgHcA7494n/5N+1s1JlcjCV0IB5OVX8I/V0SyLT6dfw5ty5wRHXArrzUb3CxUez6yCb68G9w9YdpP0KKvZc5bjTJdxttRb/Pxvo/p37Q/C4csxM/D76KfcVNuMv1HDUhCF8KK1u07zXu/x/PE6E4MrDBhVtSJTJ76Zi9ZBVUXF84tNFJoLOX1yVdxc2iLml24rAx+fwViVlDtejQ5p6FRR7j9C6hvvaHxBcYCntr6FBtObODmDjfzZL8n8XCT9m9rkYQuhBVorXl/8xFeW3cQT4Mbdy3dxUs3dOO2sJZ8H3OSx77eQ+N6XlzTvuqsiG5KMblPMKEhATW7eHEefHufafh+uxFQt3HVMnUC4Zp/gXe9ml3DDCn5Kcz+bTZx6XE8FvoYd3a5U2rdViYJXQgLKzaW8dS3e/k6MonxPZrx7Lgu/Our3TzxzV5+2JPMtvh0wkIC+ODOPgT4el76hJcj+5RpAedTu2HUK9D/AZvNkHgi+wTHs48DkG/M57Xw18gpzuGdYe8wpMUQm8RQ20lCF8LC3vntMF9HJvHQ8PbMua49SimW3h3KCz/G8sn249zUO5iXb+yGl7uF5xo5tRtW3AaFWTBlFXQcbdnzX8SaI2t47s/nznvh2cS3CZ9e/ykdAzraLI7aThK6EBZ0PD2PD7ccZWLPZswd0eHcfneDGy9M7MaMa9oQ3MDH8k0PB36C1feCTwOYvh6adLfs+S+gTJfxbvS7fLT3I8KahDG712zclGkhtLb12+Lr4WuTOISJJHQhamhP0ll+2nuKh4e3Pzevyos/xuLhpnhqTPWLd7cIsMLybdGfw/cPQrNepnnL61p4bvQKcopzeDvqbVLyUwBIK0xjT+oebmp/E0/3f1peeNqZWQldKTUaeAswAEu01guqKXML8DymV+q7tda3WzBOIRzOy2vj2HE0gz8Op7Hk7lAOnM5hQ1wKT1zficb1bLia1PZF0LSHqfuhFdf7rDjDYZv6bQBT98LH+z7OHZ3vkBeeDuCSCV0pZQAWASOAJCBcKbVGax1boUx74ElgkNY6UynlmEubCGEhx9Ly2HE0g5FdGvPnkXQmvrsNLw832gT6cs8gGy7SUpAJKbGmUZ5WTOYxKTE8vOlhSspK+GDEB/Rr2s9q1xI152ZGmTAgXmt9VGtdDKwCJlYqMwNYpLXOBNBap1g2TCEcy6rwExjcFC/d0I2vHxiAh8GNxIwCnh3fBU93c36sLCRxF6Ch1QCrXeKnoz8xff10/Dz8WDFmhSRzB2ZOk0tzILHCdhJQ+Yl2AFBKbcPULPO81npd5RMppWYCMwFatqw96/wJ11JsLGN1ZBLDOgURVM+boHrerJk1iNhT2VzTvpFtgzn+J7h5QPM+Fj+11pr3dr/HB7s/kBkOnYQ5Cb26hrHKQ8/cgfbAUCAY2KqU6qa1Pnveh7ReDCwGCA0NrWb4mhCO77cDZ0jLLWZK2N+jOBv6edk+mQOc2E5M867sjP3E4qfel76P3xN/Z2LbiTw34DmZ4dAJmJPQk4CK44+DgeRqyuzQWpcAx5RSBzEl+HCLRCmEA1m5K5Em9bwZbI8EXlFJAV9mHeDlhvUpjXnX4qd3d3Pn4d4PM73bdHnh6STMSejhQHulVGvgJHAbULkHy3fAFGCZUioQUxPMUUsGKoQjSMrMZ8vhVGZf2w53gw3byispLStl4eYn+KyhP9fU78iC6z+mjrtlX4oqVK1caNmZXTKha62NSqlZwHpM7eNLtdb7lVIvABFa6zXlx0YqpWKBUuAxrXW6NQMXwtZKyzRv/noYgFv61nDSrGoczjzMs9ueJa0wzezPFJcWk1GYwdSsbB6d/B4GT+vNySKch1n90LXWa4G1lfY9W+FrDcwt/yOEy8ktMvLQymh+O5DCA0PbEtzAMrXhLUlb+L8t/0cd9zoMan55izKHHdnBeIMv+Nq56Uc4DBkpKlyG1prdqbvJLcm16HnTc4t589eDnDxbyD9GtGJQpzSzVt25lAMZB3gn+h06NujI28PeponvZYzwLCuFLSFWX5RCOBdJ6MIllJSW8OKOF/k2/lvrXKAueNeFr5NMfyzl2hbXsuCaBdTxuMwa/5n9UJQNrQZaLhjh9CShC6eXVZTFnN/nEH46nBndZ1hsqtZt8Wm8vfEwDXw9eXpMZ4vPw+Lp5knHgI7nJrO6LCd2mP5uab0BRcL5SEIXFlVUWsR7Me+dmxfbFg5kHCAlP4VXrnmFcW3G1egcpWWaDzYfYU+SaehEkbGM3w8WEtrqKj68sw8N/bwsGfKVO/En1AuG+pZ7OSucnyR0YTHpBek8sukRYlJjaFe/nc36Lgf6BPLKNa/QK6hXjT6fV2Tk4VUxbIg7Q9tGvniUd0e8s38r/j2us+XnLb9SZaVwdDO0H2nvSISDkYQuzskpzql2RXZPg+cl57WOz4xn1m+zSCtI440hbzAyxHGTTbGxjNwi031m5hcze0U0B05nM29CV+4eGGLf4MyRFAEFGdBhlL0jEQ5GErrAWGbktfDXWHlgZbXH3ZQbD/R4gPuuuq/aWve2k9t4dPOjeLt7s2z0MroFdrN2yDWWlV/C2He2kpRZcG6fn5c7S6f1ZWhHJ5kk9NA6UAZoO8zekQgHIwm9lsspzuGxLY+x7eQ2bmp/Ex0adKhSJiolikUxi0jITmDewHl4Gf5uT151YBULdi2gXf12vDv83cvremcH//n1IMlnC3h8dCfqeJqaUgZ3aETrQCdaWefQelPvFh+ZKEucTxJ6LXY67zQPbHiAhKwEnhvwHJM7TK623JROU+jQoAPvRL9DUk4SQ4JNvUiOZR3jh6M/MCR4CK8Nfu3yu97ZWNypbD7dcZyp/VvxwNC29g6nZs4mQsp+GPmSvSMRDkgSei32wvYXSM5N5v0R79O/af8LllNKMfOqmbSq14rn/nyO3am7AVNTzN1d7mZOnzkOP+eH1prnvt+Pv4/HeWt9Op3D601/t5f2c1GVJPRaanPiZrae3MqjoY9eNJlXNCpkFNe1vI4yXWbaoXCaNSTX7E5mV0IGr9zYnfp1PO0dTs0dWg8NWkNge3tHIhyQJHQXVFJawqvhr7I1aeu5fa3rt+alQS8R6BNIUWkRr4a/Shv/Ntze+fKWfjW4GTDgOLXxTQdSeHltHPnFpRctl55XRPfm/twS6sT9tovz4dgW6DMNZDpbUQ1J6C4mszCTOb/PIfJMJMNbDsfXwxetNRtObOD2n27nnWHvsDlpM4k5iSwesdhpatiVaa1Z9mcCL/4YS9tGfvRv0/Ci5T3dFTOuaYPBzYkT4bEtYCyU7origiShu5BjWcd4cOODnMk7w6vXvMqYNmPOHZuaPpXZG2dz1893UabLGNFqBAOaOeewcWNpGfN+iOXTHccZ0aUx/721J75eTvhP+WSkaZFnADd30zB+90ojUtOPQOYx09cxn4GnH7S6vFkZRe3hhD8Fojo7T+1kzu9z8HDz4ONRH9MzqOd5x7s07MKKsSuY/dtsErITeDT0UTtFemWyC0t48PMoth5O477BbXh8dCfcnK3WXWqEdU9A+Efn72/eB25bCXUbm7ajPoEf50DFwV5dJ1VN+kKUk4TuAr4+9DXzd8wnxD+Ed4e/S3O/5tWWa+zbmM/HfE5WcRaBPoE2jvLKJWbkc5+p3CMAACAASURBVM+ycI6l5bHgxu7cFuaEC40XZsHX90D8BhgwC7pMNO1Pj4ef/gVLhsNtK2DvV/Dn26bBQ0Meh78m8ArqbL/YhcOThO5kSkpL+GDPBxw9a1rhL68kj+2ntjOo+SAWDl6In6ffRT/vYfBwymQeeTyDmZ9EYizTfDI9jIFtne8eKDgLS0dD+mEY/zb0ufvvYy3CIKgLrLwNPhwMaOh7L4x+FQzyYyrMY9a/FKXUaOAtTEvQLdFaL6h0fBrwOqY1RwHe1VovsWCcAtM0sXN/n8uu07to69/23DD8f3T7Bw/1egh3N9f8wf8u+iT/9/UemtX3Zum0vrRpdPH/tBxWzOeQGgd3rIb211U93qwnzPjNVFNvOwzCZtg+RuHULpkBlFIGYBEwAkgCwpVSa7TWsZWKfqG1nmWFGGuVQmMhhcbCKvvP5J/h0c2PcjL3JC9f/TLj2463Q3S2pbXmzQ2HeXvjYfq1DuCDqX1o4Oukfci1hsjlENy3+mT+l3rNYEr1c+oIcSnmVOnCgHit9VEApdQqYCJQOaGLK/Tj0R+Z9+c8CkurJnSABl4NWDJyCb0b97ZxZLZXWFLKY1/v4YfdydzcJ5j5k7rj6V6DhSAcReJOSDsIE961dyTChZmT0JsDiRW2k4B+1ZS7SSk1GDgEzNFaJ1YuoJSaCcwEaNnSCV9oWYnWmkUxi/hwz4f0adyHEa1GVCnjptwYGjyUpn5N7RChbaXkFDLzk0hiEs/y+OhO3D+kjc3mVreayOXgWRe63WjvSIQLMyehV/eTpCtt/wCs1FoXKaXuB5YDVeb21FovBhYDhIaGVj5HrVRoLOSZbc+wLmEdN7S7gWf7P4uHwTkH+9TU/uQsfj+Yem57xc4TpOcV8cHU3ozu5gL/gRWchf3fQo/bwNOJZnUUTsechJ4EVBwvHQwkVyygtU6vsPkR8OqVh+b60grSePi3h9mTtodHej/CPd3ucf6a6GVaszuZR7/aTbGx7Ny+5vV9+Oq+gXQP9rdjZBa09yswFpzfq0UIKzAnoYcD7ZVSrTH1YrkNOG8CEKVUU631qfLNCUCcRaN0QYcyDzFr4ywyCzN5c+ibXNfqIi/KXJDWmrc3xvPmhkOEhQTw7h298Pcx/Wbi4ebmfIOFLkRriFoOTbpD056XLi/EFbhkQtdaG5VSs4D1mLotLtVa71dKvQBEaK3XAA8ppSYARiADmGbFmJ3G14e+Ztn+ZdUu65ZWkIa/pz/Lrl9G14Zd7RCd7c3/KZaf950GoKS0jDPZRdzUO5iXb+zmeOt2WkpyNJzeC2MWyoRawurM6ristV4LrK2079kKXz8JPGnZ0JxXaVkpCyMW8lncZ/Ro1INW9VpVKePj7sOM7jNo7NvYDhHa3pZDqXy09RgD2zakib83AL1aNmBqv5au3cwUtRzcfaD7zfaORNQCrjkSxY7yS/L5vy3/x+akzUztPJV/hf7LZQf8mKvYWMbzP+wnpGEd/vePvq5bG6+sKBf2fm2af0WWixM2ULszjRXM3zmfP07+wb/7/ZtbO91q73AcwrI/j3E0NY+l00JrTzIH2P8NFOfKy1BhM048UsPxxKTEsObIGqZ1nSbJvNyZ7ELe2nCYYZ2CGNapdjQvnRP1CQR2hBbVDdsQwvKkhm4hpWWlvLzzZYLqBDHzqpn2DsdutNZ8sPkoe5LOAnAsLY+SUs2z47rYOTIbOxMLSeEwcr68DBU2IwndQlYfXk1cRhyvDX6NOh517B2O3WyMS+HVdQdoGVAHbw/TL4DPT+hKSGAtG1ATtRwMntBjir0jEbWIJPQa0lqTXZwNmKawfSf6HUIbhzI6ZLSdI7OfwpJSXvgxlvZBfqx9+Bo8DLW0Ra+kEHavgk7jwPfiS+MJYUmS0Gto/s75fHHwi3PbbsqNJ8KecO0ueJfw0ZajnMjI5/N7+9XeZF5WCr/8GwrPystQYXOS0Gsguzib7+O/Z2CzgQwOHgxAxwYd6RjQ0c6R2c/JswUs+j2e67s1YVA7J1x8whKKcuDr6XB4PfT/J7QeYu+IRC0jCb0G1h5dS2FpIQ/1fqjWjPK8lPk/mWZTfnpsLV0i7WyiabWhlDgY+4ZptSEhbEwS+mXSWrP68Go6B3SWZF7uSGoua/ee5qHh7QluUAtfCCdFmpK5sRDu+AraDbd3RKKWqqUNnTUXmx7LgYwD3NT+JnuH4jC+CE/E3U0xtX8tnON+3zewbAx4+MD0XyWZC7uSGvplWn14Nd4Gb8a0GWPvUBxCsbGM1ZFJDO8cRFBdb3uHYztaw5aFsOklaNEfbvscfGvpuwPhMCShX4b8knzWHlvLyJCR1PWsa+9wHMKvsWdIzyvmtrBaVDs3FsGa2bDnC+h+C0x4Bzxq0X9mwmFJQr8M6xPWk1eSx+QOk+0disNYFX6C5vV9GNy+kX0CKCuDkxFQnFf1mIePadj9xbqSZiZAxjHzr6fLYMvrcGI7XPtvGPyojAQVDkMSupmiU6J5M/JN2tVvR89GslABQGJGPlsPp/HIde0x2GNBipIC+O4B0/JuF3LNozD8meqP7fkSvn8QSosv77ru3jD5f7I+qHA4ktDN8MORH3juz+do5teM/17731o9eKiiL8ITUQpuCW1x6cKWlnMGVk2Bk1GmmnLI1VXL7Hwf/nwbet4ODdv+vV9r+P0V2PwqtLoarn0K1GX0D6jfEvybX/k9CGFhktAvYfn+5SyMWEhYkzD+M/Q/+Hu5yDqXVygrv4SvIhMZ0qERzer7WP+C6Udg82umroFgmviqIBNu/Qw6j6v+MwGtIX4jrHsS7vjStK+kwFQr37caek6FcW+Cu6f14xfCBsyqliilRiulDiql4pVST1yk3GSllFZKhVouRPvRWvO/ff+jf9P+fHDdB5LMyyWk5THp/W1k5BVz3+C2l/7AlSorg2/vh7g1poE7KXFQrzn84+cLJ3OAuk1gyOOmkZsH10FuCiwfb0rm1z0PE9+VZC5cyiVr6EopA7AIGAEkAeFKqTVa69hK5eoCDwE7rRGoPZzIOUF6YToPhjyIh8HD3uHYTUFxKcWlZQDsP5nFP1dEoYDP7+1PWOsA6wewZxUk7YKJ70GvOy7vs/3uh+hP4efHQAN5qXDLp9BlglVCFcKezGlyCQPitdZHAZRSq4CJQGylci8CrwGPWjRCO4o6EwVAn6A+do7EPrTW/G9bAi+vjcNYps/tb9vIl6XT+tKqoQ2mxC3Mgl+fg+C+NZuK1t0Trn8VPp0Efk3gnp+hWS/LxymEAzAnoTcHEitsJwHnLcGilOoFtNBa/6iUumBCV0rNBGYCtGzp+P2Wo1KiqO9Vn9b+re0dis2VlJbx/Jr9fL7zBMM6BZ2bcMvT3Y0JPZrh72Oj31g2v2aqVd/+BbjVcGBz22Fwx2po0s3UDCOEizInoVfXpeNcdU0p5Qa8CUy71Im01ouBxQChoaH6EsXtLupMFL2CetW6Xi15RUbu/yySrYfTuG9IGx4f1Qk3e3RLPBMLOz+A3ndC895Xdq7211kmJiEcmDlVniSgYr+0YCC5wnZdoBvwu1IqAegPrHH2F6NpBWmcyDlB76ArTCRO6M1fD/FHfBqv3XQVT17f2T7J/MQOWD4OvP1h+HO2v74QTsichB4OtFdKtVZKeQK3AWv+Oqi1ztJaB2qtQ7TWIcAOYILWOsIqEdvIX+3nvRvXroQen5LDsj8TuDW0Bbf0tUP/coDdX5h6o3jXN014JXOkCGGWSyZ0rbURmAWsB+KAL7XW+5VSLyilXLarQHRKNN4GbzoH1J75vbXWPL8mljqeBh4bZYfFOsrK4Lf58O1M05D9ezecPyBICHFRZg0s0lqvBdZW2vfsBcoOvfKw7C/yTCRXNbqqVnVXXLfvNH/EpzFvQlca+nnZ9uIVh/H3mgpjZcCPEJdLRopWI68kj4OZB5nRfYa9Q7G4guJSohMz0ZVeSWsNL/0UR6cmdbmjnw16IOWlw5m9pq/LSmHTfNMw/hEvwMCHZMIrIWpAEno1dqfspkyXuVz7eWJGPtOXh3PoTG61x90UvHFLf9ytvcBz4i5YdbupO+JfPOpcfBi/EOKSJKFXIzIlEoMy0KNRD3uHYjGRxzOZ+UkEJaVlvD2lF03qVZ2/O6iuFyGBVh4stPdr+O6fUK8Z3PA+eJZfr0GIaZ8QosYkoVdiLDOyPXk7HQM64uthg5GQVrLpYApfRyYBppedG+JSaOrvzdJpfWnbyM92gUQshWNbTF8X55vmVWk1yFQbr2ODaQOEqEUkoVeQW5zLY1seY2/aXp4Iu+AcZA4vJbuQ2Sui8XJ3o34d00vdazs2YsGNV9HA14YvGk/shB/nmCbS+qsm3ncGjJoP7jZ+6SpELSAJvdzJ3JPM2jiLY1nHeKb/M9zS8RZ7h1RjC9YdoNhYxg+zr6a1tZtQLqSsFNY+CnWbwYO7wMuGvxUIUUtJQsfUJHHfr/eRUZDB+9e9z4BmA+wdUo1FHs/gm6iT/HNoW/slc4Co5XB6D9z0sSRzIWzEyt0ZnMOZ/DMczz7Og70edOpkXlqmefb7/TSp582D17azXyD5GbDxRdNqQN1usl8cQtQyUkMHYtNNMwF3bdjVzpGY57cDZzhcTdfDhPQ89idn886UXvh62fDRag37v4Es00tYjv9pmvZ2zGvSn1wIG5KEjimhuyk3OgbYYbj7ZSgt07yyNo4lf1x4lfoRXRoz7qqmtguqpBDWzIa9X56/f/Bj0Ng5/oMUwlVIQgfiMuJo498GH3cbrI1ZQ3lFRh5eFcOGuDPcPaAVj43uRHWTIPp4GGw33W9eGqy6AxJ3mBZq7v9AeY1cgWcd28QghDhHEjqmGvqApo7bdn4qq4DpyyI4cDqbeRO6cvfAENtdPOMorJ4B+elVjxVkgLEIbl4GXSfZLiYhRLVqfUJPzU8lrSCNLg272DuUau1JOsu9yyPILy5l6bS+DO0YZLuLaw1r/w9SD0LH66seN3hA3+nQvHYu0SeEo6n1CT0uIw6Azg0db5rcdftO8cgXMQT6efHp9H50bFLXtgEcWgfxv8LI+TBwlm2vLYS4bLU+oe9P349C0Smgk71DOU9mXjGzV0bTrbk/H90VSqDNp7MthHVPQGBH6Hefba8thKiRWp/Q49LjaFWvlcPN2xJxPJOSUs2T13e2fTIH2P4OZCbAnd+ZmlaEEA6v1if02PRY+jR2vDbg8IQMPA1uXBXsb72LnNgB4R9DmbHqsUProPN4aHut9a4vhLAosxK6Umo08BZgAJZorRdUOn4/8CBQCuQCM7XWsRaO1eLSC9I5k3/GIV+IhidkcFWwP94eButcIGalqf+4V12o07Dq8aY9YdQr1rm2EMIqLpnQlVIGYBEwAkgCwpVSayol7BVa6w/Ky08A/gOMtkK8FvXXC1FHS+gFxaXsTcpixuA2NT+J1lCUU90B2PYWbH0DWg+BW5aDT4OaX0cI4TDMqaGHAfFa66MASqlVwETgXELXWmdXKO8LVFrgzDHFpZsSuqONEI1JPIuxTNM3pIaJNjcVvrwTTmy/cJk+02DMQmkfF8KFmJPQmwOJFbaTgH6VCymlHgTmAp7AsOpOpJSaCcwEaNnSButWXkJcRhwt6ragnmc9e4dynvCEDJSCPi1rsABEShysuMWU1Ic8YWpSqaxBK+g0TuZZEcLFmJPQq/upr1ID11ovAhYppW4H/g3cXU2ZxcBigNDQULvX4mPTY+kW2M3eYVQRnpBBx8Z18a9zmbXn+A3w1T/Awwf+8ZMM+BGiljFn+twkoEWF7WAg+SLlVwE3XElQtpBVlMXJ3JN0DnCsAUXG0jKijmcSernNLbs+gs9vhvqtYMZvksyFqIXMSejhQHulVGullCdwG7CmYgGlVPsKm2OBw5YL0Tr+mjLX0V6IHjidQ15xKX1DzGxuKTXC2sdMqwO1HwX3rAP/YOsGKYRwSJdsctFaG5VSs4D1mLotLtVa71dKvQBEaK3XALOUUtcBJUAm1TS3OJpzQ/4drIa+61gGgHkJXWv4ZoZpLvIBs2DEC+BmpW6OQgiHZ1Y/dK31WmBtpX3PVvj6YQvHZXVx6XE092tOfe/69g7lPBHHM2he34dm9c2YyvfQOlMyH/oUDH3c+sEJIRxarR0pGpsea9PaudaaPUlZ5BeXXrTcrmOZXN2umoE+lVWca+WauRaKUgjhzGplQs8pzuFEzgkmtptos2t+uOUoC34+YFbZgW0DL13or7lW7vpe+pILIYBamtAPZJgSq61eiJ7OKuTtjYcZ2rER9w1ue9Gynu6KHsGXaAY6mwhb3oDOE6DNUIvFKYRwbrUyof/Vw8VWTS4vr43DWKZ5YUI3Wja0wNJsv5a/vhg1/8rPJYRwGeZ0W3Q5semxNK7TmIY+ZrRVX6EdR9NZszuZ+4e0tUwyzzwO+781rd9Z3/6jbYUQjqNW1tDjMuKsukJRfrGRMg1lWvP8mv00r+/DA0Mu3tRitujPTH+H/sMy5xNCuIxal9DzS/JJyErg+tbVrJF5hQpLSnnq2718E3XyvP3v39EbH08L9A8vNZoServhUjsXQlRR6xL6gYwDaDRdAiz7QjQtt4j7Po0k8ngm0waG0Ly8H3lwAx9Gd2timYvEb4CcZLj+VcucTwjhUmpdQrfUHOiJGfms23caAI3mk+3HSc0pYtHtvRl7VdMrjrNaUZ+AbyPo4PBTzQsh7KDWJfTY9FgCfQJpVKdRjc9RZCzlzo93kpCef25f43pefHHfAHq2sNLI05zTppGhA2eBu6d1riGEcGq1MqFfaXfFJVuPkZCez8d3h9KvjamnjLe7G+4GK3Yaiv4MdCn0dvhpcoQQdlKrui0WGAs4mnX0ippbTmUV8O5v8Yzs0pjhnRvj5+WOn5e7dZN5WRlEfwoh10BDC/WWEUK4nFqV0Pel7aNMl9E9sHuNz/Hy2gOUac0z42w47e6xzaZh/lI7F0JcRK1K6FFnolAoegb1rNHntx9J54fyQUItAiwwSMhcUeULOXceb7trCiGcTu1K6ClRtGvQDn8v/8v+bPSJTGavjDYNEhpqw2aPvHSI+xGuug08vG13XSGE06k1Cd1YZiQmJYbeQb0v+7M/7E7m1sU78PUysPyeMLw9bLiIxO6VUFYCfaS5RQhxcbWml8uhzEPkG/MvmdBLyzSLtxzlwOlsAPKLS/k19gxhIQF8cGcfAnxt2GVQa1NzS3AYBDnWykpCCMdjVkJXSo0G3sK0BN0SrfWCSsfnAvcCRiAVuEdrfdzCsV6R6JRoAHo3vnBCzy0y8vDKaDYeSKFFgA8GpQC4s38r/j2uM17uNl7e7cQOSDsEExfZ9rpCCKd0yYSulDIAi4ARQBIQrpRao7WOrVAsGgjVWucrpR4AXgNutUbANRV5JpJmvs1o4lv9MPzkswXcsyycwym5vDCxK3cNCLFtgNWJWg5e9aDrJHtHIoRwAua0oYcB8Vrro1rrYmAVcN5SP1rrTVrrv4ZN7gAcatl5rTXRKdH0atyr2uO7E88ycdE2TmYWsHRaX8dI5vEbYN830O0m8PS1dzRCCCdgTkJvDiRW2E4q33ch04GfqzuglJqplIpQSkWkpqaaH+UVSsxJJK0grdr287V7T3HLh9vxcndj9T8HMqRDzacEsJhdH8Hnt0Bgexj6hL2jEUI4CXPa0FU1+3S1BZWaCoQCQ6o7rrVeDCwGCA0NrfYc1hCVEgVwXkLXWvPe70d4ff1B+rRqwOI7+9DQz8tWIVVPa1j3JOx83zQB101LwKuufWMSQjgNcxJ6EtCiwnYwkFy5kFLqOuBpYIjWusgy4VlG1Jko/L38aVO/DWCaXOvJ1Xv5Jvokk3o155Ubu9u2K+KF7P3KlMz73Q+jXgY3B4hJCOE0zEno4UB7pVRr4CRwG3B7xQJKqV7Ah8BorXWKxaO8QtEp0fQK6oWbciMjr5j7Po0gPCGTuSM6MHtYO5Sq7pcQGyvKgV+egWa9YdQr4FZrhggIISzkkglda21USs0C1mPqtrhUa71fKfUCEKG1XgO8DvgBX5UnxxNa6wlWjNtsfxw5RkJ2AqmnejAs+ncy8oopKC7lnSm9GN+jmb3D+9vm1yD3NNz2uSRzIUSNmNUPXWu9Flhbad+zFb6+zsJxWcS6faeZu/ZzDI2ho38v6tavh4fBjbsHhlhv3vKaSD0EO96HnlMhONTe0QghnJTLjBQtK9PsPZlFYUkpADuPZfCfXw/RpN1hvL0DWXbLjfZtWtEaTu2G4ryqxza/Ch514LrnbB+XEMJluERCzy82MueLGNbvP3Pe/rFXBRHJIYa2GGX/dvI/34Ffn7nw8etfA78g28UjhHA5Tp/Qz2QXcu/yCPYlZ/HYqI70Km9K8fY0UOJxiC2/5DI4eLB9g8xOht8XQLvrYOBDVY971YVm1Q96EkIIczl1Qj+ensetH+4gp7CEJXeFMrxz4/OOvx6+BQ83D/o37W+nCMv9+iyUGWHMQghobd9YhBAuy6kT+gebj3K2oJhvHhhEl2b1qhzfkrSFsCZh1PGw4WIUlR3/09S/fPBjksyFEFbltP3j8oqMrIk5ydjuzapN5sezj5OQnWCf5paSQijOh6JcWPt/4N8Crp5r+ziEELWK09bQf9yTTF5xKVPCWlR7fEvSFgDbJnRjEfw4B2I+P3//zcvB046/JQghagWnTegrdyXSLsiPPq0aVHt8c9Jm2vq3JbiujSZ+zEuHL+6AE9uh772mWjlA/ZbQZeLFPyuEEBbglAn9wOlsYhLP8u+xnavtjphdnE3k6Uju7HqnZS+cHGNqE69Cw67FkH0KJi81TXkrhBA25pQJfdWuRDwNbtzYu2rtO60gjYd+e4hSXcrIViMtd9Hoz+CHR0zre1bHrwlM+wla9LXcNYUQ4jI4XUIvLCnl2+iTjOrWpMr6ngczDjL7t9mcLTrLm9e+SbfAbld+wbIy2DgPtv0X2gyFG943jeqszNMXDB5Xfj0hhKghp0vocT8tYnXp+zQ75QPv/j297DFVxl2+xfhpWFbgSZc1FloYoqQQsk5An3/AmNclaQshHJbTJXTlG8jZuh1oG9zgvKU3NpacIr8kmdU+3Qi29EIVgx+F3neBvacPEEKIi3C6hN5zxO0w4vYq+6M2/JM2ud4E37DSDlEJIYT9Oe3AoopKy0qJSYmhd+Oqa4YKIURt4RIJPf5sPDklOdUuAi2EELWFSyT0c4tASw1dCFGLmZXQlVKjlVIHlVLxSqkq3UeUUoOVUlFKKaNSarLlw7y46DPRBNUJopmvAy0pJ4QQNnbJhK6UMgCLgOuBLsAUpVSXSsVOANOAFZYO8FK01kSmRNInqI/9F7EQQgg7MqeXSxgQr7U+CqCUWgVMBGL/KqC1Tig/VmaFGC8qOS+ZlPwUejWWBSKEELWbOU0uzYHECttJ5fsum1JqplIqQikVkZqaWpNTVBF1prz9XF6ICiFqOXMSenXtGLomF9NaL9Zah2qtQxs1alSTU1QRlRJFXY+6tKvfziLnE0IIZ2VOQk8CKk46HgwkWyecyxd1JoqeQT0xuBkuXVgIIVyYOQk9HGivlGqtlPIEbgPWWDcs82QWZnI066h0VxRCCMx4Kaq1NiqlZgHrAQOwVGu9Xyn1AhChtV6jlOoLfAs0AMYrpeZprbtaI+BvD3/L8v3LASgsLQSk/VwIIcDMuVy01muBtZX2PVvh63BMTTFW5+/lT5v6bc5tX938aro36m6LSwshhENzusm5hrUcxrCWw+wdhhBCOByXGPovhBBCEroQQrgMSehCCOEiJKELIYSLkIQuhBAuQhK6EEK4CEnoQgjhIiShCyGEi1Ba12jixCu/sFKpwPEafjwQSLNgOM6iNt53bbxnqJ33XRvvGS7/vltpraudrtZuCf1KKKUitNah9o7D1mrjfdfGe4baed+18Z7BsvctTS5CCOEiJKELIYSLcNaEvtjeAdhJbbzv2njPUDvvuzbeM1jwvp2yDV0IIURVzlpDF0IIUYkkdCGEcBFOl9CVUqOVUgeVUvFKqSfsHY81KKVaKKU2KaXilFL7lVIPl+8PUEr9qpQ6XP53A3vHamlKKYNSKlop9WP5dmul1M7ye/6ifF1bl6KUqq+U+lopdaD8mQ+oJc96Tvm/731KqZVKKW9Xe95KqaVKqRSl1L4K+6p9tsrk7fLctkcpddlrazpVQldKGYBFwPVAF2CKUqqLfaOyCiPwL611Z6A/8GD5fT4BbNRatwc2lm+7moeBuArbrwJvlt9zJjDdLlFZ11vAOq11J6AHpvt36WetlGoOPASEaq27YVqv+DZc73kvA0ZX2nehZ3s90L78z0zg/cu9mFMldCAMiNdaH9VaFwOrgIl2jsnitNantNZR5V/nYPoBb47pXpeXF1sO3GCfCK1DKRUMjAWWlG8rYBjwdXkRV7znesBg4GMArXWx1vosLv6sy7kDPkopd6AOcAoXe95a6y1ARqXdF3q2E4FPtMkOoL5SqunlXM/ZEnpzILHCdlL5PpellAoBegE7gcZa61NgSvpAkP0is4r/Av8HlJVvNwTOaq2N5duu+LzbAKnA/8qbmpYopXxx8WettT4JLAROYErkWUAkrv+84cLP9orzm7MldFXNPpftd6mU8gNWA49orbPtHY81KaXGASla68iKu6sp6mrP2x3oDbyvte4F5OFizSvVKW83ngi0BpoBvpiaHCpzted9MVf8793ZEnoS0KLCdjCQbKdYrEop5YEpmX+utf6mfPeZv34FK/87xV7xWcEgYIJSKgFTU9owTDX2+uW/koNrPu8kIElrvbN8+2tMCd6VnzXAdcAxrXWq1roE+AYYiOs/b7jws73i/OZsCT0caF/+JtwT00uUNXaOyeLK244/BuK01v+pcGgNcHf513cD39s6NmvRWj+ptQ7WWodgeq6/aa3vADYBk8uLudQ9A2itTwOJSqmO5buGA7G4xxIlnAAAANFJREFU8LMudwLor5SqU/7v/a/7dunnXe5Cz3YNcFd5b5f+QNZfTTNm01o71R9gDHAIOAI8be94rHSPV2P6VWsPEFP+ZwymNuWNwOHyvwPsHauV7n8o8GP5122AXUA88BXgZe/4rHC/PYGI8uf9HdCgNjxrYB5wANgHfAp4udrzBlZiekdQgqkGPv1CzxZTk8ui8ty2F1MPoMu6ngz9F0IIF+FsTS5CCCEuQBK6EEK4CEnoQgjhIiShCyGEi5CELoQQLkISuhBCuAhJ6EII4SL+H97mPk/gV/D3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(test_scores, label='ReLU')\n",
    "plt.plot(test_scores2, label='Sigmoid')\n",
    "plt.plot(test_scores3, label='Tanh')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
